From 9791a506c22b2e1b52dc9fb333b6f010d57f9eda Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Tue, 4 Nov 2025 03:07:55 +0800 Subject: [PATCH 001/417] gh-140889: Test tailcall and JIT in CI (GH-140891) --- .github/workflows/jit.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 151b17e8442..40d8b74e982 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -183,3 +183,27 @@ jobs: - name: Run tests without optimizations run: | PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + tail-call-jit: + name: JIT with tail calling interpreter + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 19 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT and tailcall + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + make all --jobs 4 From cf1a2c1ee46b62fb3a5938fdfe90b7a9df312c3a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 3 Nov 2025 21:26:09 +0200 Subject: [PATCH 002/417] gh-133600: Run `Tools/wasm/wasi` on CI instead of deprecated `Tools/wasm/wasi.py` (#140907) --- .github/workflows/reusable-wasi.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 7c40e566b2a..18feb564822 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -60,24 +60,24 @@ jobs: with: path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # Include the hash of `Tools/wasm/wasi/__main__.py` as it may change the environment variables. # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python + run: python3 Tools/wasm/wasi make-build-python - name: "Restore host config.cache" uses: actions/cache@v4 with: path: ${{ env.CROSS_BUILD_WASI }}/config.cache # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + run: python3 Tools/wasm/wasi configure-host -- --config-cache - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host + run: python3 Tools/wasm/wasi make-host - name: "Display build info" run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" From c98c5b344941c45e50dded708177f7ec2e225b2b Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 3 Nov 2025 11:36:37 -0800 Subject: [PATCH 003/417] gh-131253: free-threaded build support for pystats (gh-137189) Allow the --enable-pystats build option to be used with free-threading. The stats are now stored on a per-interpreter basis, rather than process global. For free-threaded builds, the stats structure is allocated per-thread and then periodically merged into the per-interpreter stats structure (on thread exit or when the reporting function is called). Most of the pystats related code has be moved into the file Python/pystats.c. --- Include/cpython/pystate.h | 24 + Include/cpython/pystats.h | 64 +- Include/internal/pycore_interp_structs.h | 14 +- Include/internal/pycore_pystats.h | 2 +- Include/internal/pycore_stats.h | 85 +- Include/internal/pycore_tstate.h | 6 + Lib/test/test_pystats.py | 215 +++++ Makefile.pre.in | 1 + ...-07-29-17-51-14.gh-issue-131253.GpRjWy.rst | 1 + Modules/_xxtestfuzz/fuzzer.c | 4 +- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + Python/ceval_macros.h | 3 +- Python/gc.c | 7 +- Python/gc_free_threading.c | 16 +- Python/initconfig.c | 6 - Python/lock.c | 3 + Python/pylifecycle.c | 17 + Python/pystate.c | 29 + Python/pystats.c | 819 ++++++++++++++++++ Python/qsbr.c | 3 +- Python/specialize.c | 426 +-------- Python/sysmodule.c | 4 +- 24 files changed, 1269 insertions(+), 485 deletions(-) create mode 100644 Lib/test/test_pystats.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst create mode 100644 Python/pystats.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ac8798ff612..dd2ea1202b3 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -217,6 +217,15 @@ struct _ts { */ PyObject *threading_local_sentinel; _PyRemoteDebuggerSupport remote_debugger_support; + +#ifdef Py_STATS + // Pointer to PyStats structure, NULL if recording is off. For the + // free-threaded build, the structure is per-thread (stored as a pointer + // in _PyThreadStateImpl). For the default build, the structure is stored + // in the PyInterpreterState structure (threads do not have their own + // structure and all share the same per-interpreter structure). + PyStats *pystats; +#endif }; /* other API */ @@ -239,6 +248,21 @@ PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); // function is set, otherwise disable them. PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); +#ifdef Py_STATS +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +extern _Py_thread_local PyThreadState *_Py_tss_tstate; + +static inline PyStats* +_PyThreadState_GetStatsFast(void) +{ + if (_Py_tss_tstate == NULL) { + return NULL; // no attached thread state + } + return _Py_tss_tstate->pystats; +} +#endif +#endif // Py_STATS + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f..d0a925a3055 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -4,7 +4,7 @@ // // - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() // and Py_DECREF(). -// - _Py_stats variable +// - _PyStats_GET() // // Functions of the sys module: // @@ -14,7 +14,7 @@ // - sys._stats_dump() // // Python must be built with ./configure --enable-pystats to define the -// Py_STATS macro. +// _PyStats_GET() macro. // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. @@ -109,6 +109,18 @@ typedef struct _gc_stats { uint64_t objects_not_transitively_reachable; } GCStats; +#ifdef Py_GIL_DISABLED +// stats specific to free-threaded build +typedef struct _ft_stats { + // number of times interpreter had to spin or park when trying to acquire a mutex + uint64_t mutex_sleeps; + // number of times that the QSBR mechanism polled (compute read sequence value) + uint64_t qsbr_polls; + // number of times stop-the-world mechanism was used + uint64_t world_stops; +} FTStats; +#endif + typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; @@ -173,22 +185,48 @@ typedef struct _stats { CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; +#ifdef Py_GIL_DISABLED + FTStats ft_stats; +#endif RareEventStats rare_event_stats; - GCStats *gc_stats; + GCStats gc_stats[3]; // must match NUM_GENERATIONS } PyStats; +// Export for most shared extensions +PyAPI_FUNC(PyStats *) _PyStats_GetLocal(void); -// Export for shared extensions like 'math' -PyAPI_DATA(PyStats*) _Py_stats; +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +// use inline function version defined in cpython/pystate.h +static inline PyStats *_PyThreadState_GetStatsFast(void); +#define _PyStats_GET _PyThreadState_GetStatsFast +#else +#define _PyStats_GET _PyStats_GetLocal +#endif + +#define _Py_STATS_EXPR(expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL) { \ + s->expr; \ + } \ + } while (0) + +#define _Py_STATS_COND_EXPR(cond, expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL && (cond)) { \ + s->expr; \ + } \ + } while (0) #ifdef _PY_INTERPRETER -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_decrefs++) #else -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_decrefs++) #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9cdaa950e34..e8cbe9d894e 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -199,7 +199,7 @@ enum _GCPhase { }; /* If we change this, we need to change the default value in the - signature of gc.collect. */ + signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 struct _gc_runtime_state { @@ -963,6 +963,18 @@ struct _is { # ifdef Py_STACKREF_CLOSE_DEBUG _Py_hashtable_t *closed_stackrefs_table; # endif +#endif + +#ifdef Py_STATS + // true if recording of pystats is on, this is used when new threads + // are created to decide if recording should be on for them + int pystats_enabled; + // allocated when (and if) stats are first enabled + PyStats *pystats_struct; +#ifdef Py_GIL_DISABLED + // held when pystats related interpreter state is being updated + PyMutex pystats_mutex; +#endif #endif /* the initial PyInterpreterState.threads.head */ diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h index f8af398a560..50ab21aa0f1 100644 --- a/Include/internal/pycore_pystats.h +++ b/Include/internal/pycore_pystats.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef Py_STATS -extern void _Py_StatsOn(void); +extern int _Py_StatsOn(void); extern void _Py_StatsOff(void); extern void _Py_StatsClear(void); extern int _Py_PrintSpecializationStats(int to_file); diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index 24f239a2135..850e6ea4552 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -15,39 +15,56 @@ extern "C" { #include "pycore_bitutils.h" // _Py_bit_length -#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) -#define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) -#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) -#define UOP_PAIR_INC(uopcode, lastuop) \ - do { \ - if (lastuop && _Py_stats) { \ - _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ - } \ - lastuop = uopcode; \ - } while (0) -#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) -#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) -#define OPT_HIST(length, name) \ +#define STAT_INC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name++) +#define STAT_DEC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name--) +#define OPCODE_EXE_INC(opname) _Py_STATS_EXPR(opcode_stats[opname].execution_count++) +#define CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.name++) +#define OBJECT_STAT_INC(name) _Py_STATS_EXPR(object_stats.name++) +#define OBJECT_STAT_INC_COND(name, cond) _Py_STATS_COND_EXPR(cond, object_stats.name++) +#define EVAL_CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.eval_calls[name]++) +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) _Py_STATS_COND_EXPR(PyFunction_Check(callable), call_stats.eval_calls[name]++) +#define GC_STAT_ADD(gen, name, n) _Py_STATS_EXPR(gc_stats[(gen)].name += (n)) +#define OPT_STAT_INC(name) _Py_STATS_EXPR(optimization_stats.name++) +#define OPT_STAT_ADD(name, n) _Py_STATS_EXPR(optimization_stats.name += (n)) +#define UOP_STAT_INC(opname, name) \ do { \ - if (_Py_stats) { \ - int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ - bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ - _Py_stats->optimization_stats.name[bucket]++; \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + assert(opname < 512); \ + s->optimization_stats.opcode[opname].name++; \ } \ } while (0) -#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) -#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (lastuop && s) { \ + s->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.unsupported_opcode[opname]++) +#define OPT_ERROR_IN_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.error_in_opcode[opname]++) +#define OPT_HIST(length, name) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ + bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ + s->optimization_stats.name[bucket]++; \ + } \ + } while (0) +#define RARE_EVENT_STAT_INC(name) _Py_STATS_EXPR(rare_event_stats.name++) +#define OPCODE_DEFERRED_INC(opname) _Py_STATS_COND_EXPR(opcode==opname, opcode_stats[opname].specialization.deferred++) + +#ifdef Py_GIL_DISABLED +#define FT_STAT_MUTEX_SLEEP_INC() _Py_STATS_EXPR(ft_stats.mutex_sleeps++) +#define FT_STAT_QSBR_POLL_INC() _Py_STATS_EXPR(ft_stats.qsbr_polls++) +#define FT_STAT_WORLD_STOP_INC() _Py_STATS_EXPR(ft_stats.world_stops++) +#else +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() +#endif // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -71,6 +88,9 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) #define OPCODE_DEFERRED_INC(opname) ((void)0) +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() #endif // !Py_STATS @@ -90,6 +110,11 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); RARE_EVENT_INTERP_INC(interp, name); \ } while (0); \ +PyStatus _PyStats_InterpInit(PyInterpreterState *); +bool _PyStats_ThreadInit(PyInterpreterState *, _PyThreadStateImpl *); +void _PyStats_ThreadFini(_PyThreadStateImpl *); +void _PyStats_Attach(_PyThreadStateImpl *); +void _PyStats_Detach(_PyThreadStateImpl *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c7..29ebdfd7e01 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -70,8 +70,14 @@ typedef struct _PyThreadStateImpl { // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; + +#ifdef Py_STATS + // per-thread stats, will be merged into interp->pystats_struct + PyStats *pystats_struct; // allocated by _PyStats_ThreadInit() #endif +#endif // Py_GIL_DISABLED + #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif diff --git a/Lib/test/test_pystats.py b/Lib/test/test_pystats.py new file mode 100644 index 00000000000..c50cecfcfdd --- /dev/null +++ b/Lib/test/test_pystats.py @@ -0,0 +1,215 @@ +import sys +import textwrap +import unittest +from test.support import script_helper + +# This function is available for the --enable-pystats config. +HAVE_PYSTATS = hasattr(sys, '_stats_on') + +TEST_TEMPLATE = """ + import sys + import threading + import time + + THREADS = 2 + + class A: + pass + + class B: + pass + + def modify_class(): + # This is used as a rare event we can assume doesn't happen unless we do it. + # It increments the "Rare event (set_class)" count. + a = A() + a.__class__ = B + + TURNED_ON = False + def stats_on(): + global TURNED_ON + sys._stats_on() + TURNED_ON = True + + TURNED_OFF = False + def stats_off(): + global TURNED_OFF + sys._stats_off() + TURNED_OFF = True + + CLEARED = False + def stats_clear(): + global CLEARED + sys._stats_clear() + CLEARED = True + + def func_start(): + pass + + def func_end(): + pass + + def func_test(thread_id): + pass + + _TEST_CODE_ + + func_start() + threads = [] + for i in range(THREADS): + t = threading.Thread(target=func_test, args=(i,)) + threads.append(t) + t.start() + for t in threads: + t.join() + func_end() + """ + + +def run_test_code( + test_code, + args=[], + env_vars=None, +): + """Run test code and return the value of the "set_class" stats counter. + """ + code = textwrap.dedent(TEST_TEMPLATE) + code = code.replace('_TEST_CODE_', textwrap.dedent(test_code)) + script_args = args + ['-c', code] + env_vars = env_vars or {} + res, _ = script_helper.run_python_until_end(*script_args, **env_vars) + stderr = res.err.decode("ascii", "backslashreplace") + for line in stderr.split('\n'): + if 'Rare event (set_class)' in line: + label, _, value = line.partition(':') + return value.strip() + return '' + + +@unittest.skipUnless(HAVE_PYSTATS, "requires pystats build option") +class TestPyStats(unittest.TestCase): + """Tests for pystats functionality (requires --enable-pystats build + option). + """ + + def test_stats_toggle_on(self): + """Check the toggle on functionality. + """ + code = """ + def func_start(): + modify_class() + """ + + # If turned on with command line flag, should get one count. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '1') + + # If turned on with env var, should get one count. + stat_count = run_test_code(code, env_vars={'PYTHONSTATS': '1'}) + self.assertEqual(stat_count, '1') + + # If not turned on, should be no counts. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '') + + code = """ + def func_start(): + modify_class() + sys._stats_on() + modify_class() + """ + # Not initially turned on but enabled by sys._stats_on(), should get + # one count. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '1') + + def test_stats_toggle_on_thread(self): + """Check the toggle on functionality when threads are used. + """ + code = """ + def func_test(thread_id): + if thread_id == 0: + modify_class() + stats_on() + modify_class() + else: + while not TURNED_ON: + pass + modify_class() + """ + # Turning on in one thread will count in other thread. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '2') + + code = """ + def func_test(thread_id): + if thread_id == 0: + modify_class() + stats_off() + modify_class() + else: + while not TURNED_OFF: + pass + modify_class() + """ + # Turning off in one thread will not count in other threads. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '1') + + def test_thread_exit_merge(self): + """Check that per-thread stats (when free-threading enabled) are merged. + """ + code = """ + def func_test(thread_id): + modify_class() + if thread_id == 0: + raise SystemExit + """ + # Stats from a thread exiting early should still be counted. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '2') + + def test_stats_dump(self): + """Check that sys._stats_dump() works. + """ + code = """ + def func_test(thread_id): + if thread_id == 0: + stats_on() + else: + while not TURNED_ON: + pass + modify_class() + sys._stats_dump() + stats_off() + """ + # Stats from a thread exiting early should still be counted. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '1') + + def test_stats_clear(self): + """Check that sys._stats_clear() works. + """ + code = """ + ready = False + def func_test(thread_id): + global ready + if thread_id == 0: + stats_on() + modify_class() + while not ready: + pass # wait until other thread has called modify_class() + stats_clear() # clears stats for all threads + else: + while not TURNED_ON: + pass + modify_class() + ready = True + """ + # Clearing stats will clear for all threads + stat_count = run_test_code(code) + self.assertEqual(stat_count, '0') + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 656d9dacd96..dd28ff5d2a3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -483,6 +483,7 @@ PYTHON_OBJS= \ Python/pylifecycle.o \ Python/pymath.o \ Python/pystate.o \ + Python/pystats.o \ Python/pythonrun.o \ Python/pytime.o \ Python/qsbr.o \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst new file mode 100644 index 00000000000..2826fad2330 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst @@ -0,0 +1 @@ +Support the ``--enable-pystats`` build option for the free-threaded build. diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index a04f1412eef..0cbe10c79ab 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -10,8 +10,8 @@ See the source code for LLVMFuzzerTestOneInput for details. */ -#ifndef Py_BUILD_CORE -# define Py_BUILD_CORE 1 +#ifndef Py_BUILD_CORE_MODULE +# define Py_BUILD_CORE_MODULE 1 #endif #include diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index e65f201623f..605861ad3fd 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -257,6 +257,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index a9fb6f2328a..c67fe53363e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -367,6 +367,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index a50ffb120bc..359a47fbfc4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -658,6 +658,7 @@ + diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 868ab6f7558..afdcbc563b2 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -62,8 +62,9 @@ #ifdef Py_STATS #define INSTRUCTION_STATS(op) \ do { \ + PyStats *s = _PyStats_GET(); \ OPCODE_EXE_INC(op); \ - if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ + if (s) s->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else diff --git a/Python/gc.c b/Python/gc.c index a1f3d86d910..03a5d7366ea 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2111,10 +2111,11 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) _PyErr_SetRaisedException(tstate, exc); GC_STAT_ADD(generation, objects_collected, stats.collected); #ifdef Py_STATS - if (_Py_stats) { + PyStats *s = _PyStats_GET(); + if (s) { GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + s->object_stats.object_visits); + s->object_stats.object_visits = 0; } #endif validate_spaces(gcstate); diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 842aa340154..f39793c3eeb 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2362,8 +2362,9 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) assert(generation >= 0 && generation < NUM_GENERATIONS); #ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; + PyStats *s = _PyStats_GET(); + if (s) { + s->object_stats.object_visits = 0; } #endif GC_STAT_ADD(generation, collections, 1); @@ -2426,10 +2427,13 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) GC_STAT_ADD(generation, objects_collected, m); #ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + { + PyStats *s = _PyStats_GET(); + if (s) { + GC_STAT_ADD(generation, object_visits, + s->object_stats.object_visits); + s->object_stats.object_visits = 0; + } } #endif diff --git a/Python/initconfig.c b/Python/initconfig.c index 5dc68eb4ec2..7176670c110 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2810,12 +2810,6 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } -#ifdef Py_STATS - if (config->_pystats) { - _Py_StatsOn(); - } -#endif - return _PyStatus_OK(); } diff --git a/Python/lock.c b/Python/lock.c index 98f3f89c201..789065d8162 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -6,6 +6,7 @@ #include "pycore_parking_lot.h" #include "pycore_semaphore.h" #include "pycore_time.h" // _PyTime_Add() +#include "pycore_stats.h" // FT_STAT_MUTEX_SLEEP_INC() #ifdef MS_WINDOWS # ifndef WIN32_LEAN_AND_MEAN @@ -62,6 +63,8 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } + FT_STAT_MUTEX_SLEEP_INC(); + PyTime_t now; // silently ignore error: cannot report error to the caller (void)PyTime_MonotonicRaw(&now); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8fcb31cfd12..805805ef188 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -26,6 +26,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include "pycore_typeobject.h" // _PyTypes_InitTypes() @@ -656,6 +657,14 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. @@ -2469,6 +2478,14 @@ new_interpreter(PyThreadState **tstate_p, return status; } +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. diff --git a/Python/pystate.c b/Python/pystate.c index 24681536797..cf251c120d7 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -21,6 +21,7 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_stackref.h" // Py_STACKREF_DEBUG +#include "pycore_stats.h" // FT_STAT_WORLD_STOP_INC() #include "pycore_time.h" // _PyTime_Init() #include "pycore_uop.h" // UOP_BUFFER_SIZE #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -465,6 +466,12 @@ alloc_interpreter(void) static void free_interpreter(PyInterpreterState *interp) { +#ifdef Py_STATS + if (interp->pystats_struct) { + PyMem_RawFree(interp->pystats_struct); + interp->pystats_struct = NULL; + } +#endif // The main interpreter is statically allocated so // should not be freed. if (interp != &_PyRuntime._main_interpreter) { @@ -1407,6 +1414,9 @@ static void free_threadstate(_PyThreadStateImpl *tstate) { PyInterpreterState *interp = tstate->base.interp; +#ifdef Py_STATS + _PyStats_ThreadFini(tstate); +#endif // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. if (tstate == &interp->_initial_thread) { @@ -1535,6 +1545,13 @@ new_threadstate(PyInterpreterState *interp, int whence) return NULL; } #endif +#ifdef Py_STATS + // The PyStats structure is quite large and is allocated separated from tstate. + if (!_PyStats_ThreadInit(interp, tstate)) { + free_threadstate(tstate); + return NULL; + } +#endif /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(interp->runtime); @@ -1846,6 +1863,9 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_EnsureTstateNotNULL(tstate); #ifdef Py_GIL_DISABLED _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif +#ifdef Py_STATS + _PyStats_Detach((_PyThreadStateImpl *)tstate); #endif current_fast_clear(tstate->interp->runtime); tstate_delete_common(tstate, 1); // release GIL as part of call @@ -2020,6 +2040,10 @@ tstate_deactivate(PyThreadState *tstate) assert(tstate_is_bound(tstate)); assert(tstate->_status.active); +#if Py_STATS + _PyStats_Detach((_PyThreadStateImpl *)tstate); +#endif + tstate->_status.active = 0; // We do not unbind the gilstate tstate here. @@ -2123,6 +2147,10 @@ _PyThreadState_Attach(PyThreadState *tstate) _PyCriticalSection_Resume(tstate); } +#ifdef Py_STATS + _PyStats_Attach((_PyThreadStateImpl *)tstate); +#endif + #if defined(Py_DEBUG) errno = err; #endif @@ -2272,6 +2300,7 @@ stop_the_world(struct _stoptheworld_state *stw) stw->thread_countdown = 0; stw->stop_event = (PyEvent){0}; // zero-initialize (unset) stw->requester = _PyThreadState_GET(); // may be NULL + FT_STAT_WORLD_STOP_INC(); _Py_FOR_EACH_STW_INTERP(stw, i) { _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { diff --git a/Python/pystats.c b/Python/pystats.c new file mode 100644 index 00000000000..2e377b8e6da --- /dev/null +++ b/Python/pystats.c @@ -0,0 +1,819 @@ +#include "Python.h" + +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() +#include "pycore_tstate.h" +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_uop_metadata.h" // _PyOpcode_uop_name +#include "pycore_uop_ids.h" // MAX_UOP_ID +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // NUM_GENERATIONS + +#include // rand() + +#ifdef Py_STATS + +PyStats * +_PyStats_GetLocal(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate) { + return tstate->pystats; + } + return NULL; +} + +#ifdef Py_GIL_DISABLED +#define STATS_LOCK(interp) PyMutex_Lock(&interp->pystats_mutex) +#define STATS_UNLOCK(interp) PyMutex_Unlock(&interp->pystats_mutex) +#else +#define STATS_LOCK(interp) +#define STATS_UNLOCK(interp) +#endif + + +#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#endif + +#define ADD_STAT_TO_DICT(res, field) \ + do { \ + PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ + if (val == NULL) { \ + Py_DECREF(res); \ + return NULL; \ + } \ + if (PyDict_SetItemString(res, #field, val) == -1) { \ + Py_DECREF(res); \ + Py_DECREF(val); \ + return NULL; \ + } \ + Py_DECREF(val); \ + } while(0); + +static PyObject* +stats_to_dict(SpecializationStats *stats) +{ + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + ADD_STAT_TO_DICT(res, success); + ADD_STAT_TO_DICT(res, failure); + ADD_STAT_TO_DICT(res, hit); + ADD_STAT_TO_DICT(res, deferred); + ADD_STAT_TO_DICT(res, miss); + ADD_STAT_TO_DICT(res, deopt); + PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); + if (failure_kinds == NULL) { + Py_DECREF(res); + return NULL; + } + for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { + PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); + if (stat == NULL) { + Py_DECREF(res); + Py_DECREF(failure_kinds); + return NULL; + } + PyTuple_SET_ITEM(failure_kinds, i, stat); + } + if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { + Py_DECREF(res); + Py_DECREF(failure_kinds); + return NULL; + } + Py_DECREF(failure_kinds); + return res; +} +#undef ADD_STAT_TO_DICT + +static int +add_stat_dict( + PyStats *src, + PyObject *res, + int opcode, + const char *name) { + + SpecializationStats *stats = &src->opcode_stats[opcode].specialization; + PyObject *d = stats_to_dict(stats); + if (d == NULL) { + return -1; + } + int err = PyDict_SetItemString(res, name, d); + Py_DECREF(d); + return err; +} + +PyObject* +_Py_GetSpecializationStats(void) { + PyThreadState *tstate = _PyThreadState_GET(); + PyStats *src = FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct); + if (src == NULL) { + Py_RETURN_NONE; + } + PyObject *stats = PyDict_New(); + if (stats == NULL) { + return NULL; + } + int err = 0; + err += add_stat_dict(src, stats, CONTAINS_OP, "contains_op"); + err += add_stat_dict(src, stats, LOAD_SUPER_ATTR, "load_super_attr"); + err += add_stat_dict(src, stats, LOAD_ATTR, "load_attr"); + err += add_stat_dict(src, stats, LOAD_GLOBAL, "load_global"); + err += add_stat_dict(src, stats, STORE_SUBSCR, "store_subscr"); + err += add_stat_dict(src, stats, STORE_ATTR, "store_attr"); + err += add_stat_dict(src, stats, JUMP_BACKWARD, "jump_backward"); + err += add_stat_dict(src, stats, CALL, "call"); + err += add_stat_dict(src, stats, CALL_KW, "call_kw"); + err += add_stat_dict(src, stats, BINARY_OP, "binary_op"); + err += add_stat_dict(src, stats, COMPARE_OP, "compare_op"); + err += add_stat_dict(src, stats, UNPACK_SEQUENCE, "unpack_sequence"); + err += add_stat_dict(src, stats, FOR_ITER, "for_iter"); + err += add_stat_dict(src, stats, TO_BOOL, "to_bool"); + err += add_stat_dict(src, stats, SEND, "send"); + if (err < 0) { + Py_DECREF(stats); + return NULL; + } + return stats; +} + + +#define PRINT_STAT(i, field) \ + if (stats[i].field) { \ + fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \ + } + +static void +print_spec_stats(FILE *out, OpcodeStats *stats) +{ + /* Mark some opcodes as specializable for stats, + * even though we don't specialize them yet. */ + fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); + fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); + fprintf(out, "opcode[GET_ITER].specializable : 1\n"); + for (int i = 0; i < 256; i++) { + if (_PyOpcode_Caches[i]) { + /* Ignore jumps as they cannot be specialized */ + switch (i) { + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case JUMP_BACKWARD: + break; + default: + fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); + } + } + PRINT_STAT(i, specialization.success); + PRINT_STAT(i, specialization.failure); + PRINT_STAT(i, specialization.hit); + PRINT_STAT(i, specialization.deferred); + PRINT_STAT(i, specialization.miss); + PRINT_STAT(i, specialization.deopt); + PRINT_STAT(i, execution_count); + for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { + uint64_t val = stats[i].specialization.failure_kinds[j]; + if (val) { + fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" + PRIu64 "\n", _PyOpcode_OpName[i], j, val); + } + } + for (int j = 0; j < 256; j++) { + if (stats[i].pair_count[j]) { + fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); + } + } + } +} +#undef PRINT_STAT + + +static void +print_call_stats(FILE *out, CallStats *stats) +{ + fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); + fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); + fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); + fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); + } +} + +static void +print_object_stats(FILE *out, ObjectStats *stats) +{ + fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); + fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); + fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); + fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); + fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); + fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); + fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); + fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); + fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); + fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); + fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); + fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); + fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); + fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); + fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); + fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); + fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); + fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); + fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); + fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); + fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); + fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); +} + +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); + } +} + +#ifdef _Py_TIER2 +static void +print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) +{ + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); + } +} + +extern const char *_PyUOpName(int index); + +static void +print_optimization_stats(FILE *out, OptimizationStats *stats) +{ + fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); + fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); + fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); + fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); + fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); + fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); + fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); + fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); + fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); + fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); + + print_histogram(out, "Trace length", stats->trace_length_hist); + print_histogram(out, "Trace run length", stats->trace_run_length_hist); + print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); + + fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); + fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); + fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", + stats->optimizer_failure_reason_no_memory); + fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); + fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); + for (int i = 0; i <= MAX_UOP_ID; i++) { + if (stats->opcode[i].execution_count) { + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); + } + if (stats->opcode[i].miss) { + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); + } + } + for (int i = 0; i < 256; i++) { + if (stats->unsupported_opcode[i]) { + fprintf( + out, + "unsupported_opcode[%s].count : %" PRIu64 "\n", + _PyOpcode_OpName[i], + stats->unsupported_opcode[i] + ); + } + } + + for (int i = 1; i <= MAX_UOP_ID; i++){ + for (int j = 1; j <= MAX_UOP_ID; j++) { + if (stats->opcode[i].pair_count[j]) { + fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); + } + } + } + for (int i = 0; i < MAX_UOP_ID; i++) { + if (stats->error_in_opcode[i]) { + fprintf( + out, + "error_in_opcode[%s].count : %" PRIu64 "\n", + _PyUOpName(i), + stats->error_in_opcode[i] + ); + } + } + fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size); + fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size); + fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size); + fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size); + fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size); + fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size); + + print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist); +} +#endif + +#ifdef Py_GIL_DISABLED +static void +print_ft_stats(FILE *out, FTStats *stats) +{ + fprintf(out, "Mutex sleeps (mutex_sleeps): %" PRIu64 "\n", stats->mutex_sleeps); + fprintf(out, "QSBR polls (qsbr_polls): %" PRIu64 "\n", stats->qsbr_polls); + fprintf(out, "World stops (world_stops): %" PRIu64 "\n", stats->world_stops); +} +#endif + +static void +print_rare_event_stats(FILE *out, RareEventStats *stats) +{ + fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); + fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); + fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); + fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); + fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); + fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); + fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); +} + +static void +print_stats(FILE *out, PyStats *stats) +{ + print_spec_stats(out, stats->opcode_stats); + print_call_stats(out, &stats->call_stats); + print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 + print_optimization_stats(out, &stats->optimization_stats); +#endif +#ifdef Py_GIL_DISABLED + print_ft_stats(out, &stats->ft_stats); +#endif + print_rare_event_stats(out, &stats->rare_event_stats); +} + +#ifdef Py_GIL_DISABLED + +static void +merge_specialization_stats(SpecializationStats *dest, const SpecializationStats *src) +{ + dest->success += src->success; + dest->failure += src->failure; + dest->hit += src->hit; + dest->deferred += src->deferred; + dest->miss += src->miss; + dest->deopt += src->deopt; + for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { + dest->failure_kinds[i] += src->failure_kinds[i]; + } +} + +static void +merge_opcode_stats_array(OpcodeStats *dest, const OpcodeStats *src) +{ + for (int i = 0; i < 256; i++) { + merge_specialization_stats(&dest[i].specialization, &src[i].specialization); + dest[i].execution_count += src[i].execution_count; + for (int j = 0; j < 256; j++) { + dest[i].pair_count[j] += src[i].pair_count[j]; + } + } +} + +static void +merge_call_stats(CallStats *dest, const CallStats *src) +{ + dest->inlined_py_calls += src->inlined_py_calls; + dest->pyeval_calls += src->pyeval_calls; + dest->frames_pushed += src->frames_pushed; + dest->frame_objects_created += src->frame_objects_created; + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + dest->eval_calls[i] += src->eval_calls[i]; + } +} + +static void +merge_object_stats(ObjectStats *dest, const ObjectStats *src) +{ + dest->increfs += src->increfs; + dest->decrefs += src->decrefs; + dest->interpreter_increfs += src->interpreter_increfs; + dest->interpreter_decrefs += src->interpreter_decrefs; + dest->immortal_increfs += src->immortal_increfs; + dest->immortal_decrefs += src->immortal_decrefs; + dest->interpreter_immortal_increfs += src->interpreter_immortal_increfs; + dest->interpreter_immortal_decrefs += src->interpreter_immortal_decrefs; + dest->allocations += src->allocations; + dest->allocations512 += src->allocations512; + dest->allocations4k += src->allocations4k; + dest->allocations_big += src->allocations_big; + dest->frees += src->frees; + dest->to_freelist += src->to_freelist; + dest->from_freelist += src->from_freelist; + dest->inline_values += src->inline_values; + dest->dict_materialized_on_request += src->dict_materialized_on_request; + dest->dict_materialized_new_key += src->dict_materialized_new_key; + dest->dict_materialized_too_big += src->dict_materialized_too_big; + dest->dict_materialized_str_subclass += src->dict_materialized_str_subclass; + dest->type_cache_hits += src->type_cache_hits; + dest->type_cache_misses += src->type_cache_misses; + dest->type_cache_dunder_hits += src->type_cache_dunder_hits; + dest->type_cache_dunder_misses += src->type_cache_dunder_misses; + dest->type_cache_collisions += src->type_cache_collisions; + dest->object_visits += src->object_visits; +} + +static void +merge_uop_stats_array(UOpStats *dest, const UOpStats *src) +{ + for (int i = 0; i <= PYSTATS_MAX_UOP_ID; i++) { + dest[i].execution_count += src[i].execution_count; + dest[i].miss += src[i].miss; + for (int j = 0; j <= PYSTATS_MAX_UOP_ID; j++) { + dest[i].pair_count[j] += src[i].pair_count[j]; + } + } +} + +static void +merge_optimization_stats(OptimizationStats *dest, const OptimizationStats *src) +{ + dest->attempts += src->attempts; + dest->traces_created += src->traces_created; + dest->traces_executed += src->traces_executed; + dest->uops_executed += src->uops_executed; + dest->trace_stack_overflow += src->trace_stack_overflow; + dest->trace_stack_underflow += src->trace_stack_underflow; + dest->trace_too_long += src->trace_too_long; + dest->trace_too_short += src->trace_too_short; + dest->inner_loop += src->inner_loop; + dest->recursive_call += src->recursive_call; + dest->low_confidence += src->low_confidence; + dest->unknown_callee += src->unknown_callee; + dest->executors_invalidated += src->executors_invalidated; + dest->optimizer_attempts += src->optimizer_attempts; + dest->optimizer_successes += src->optimizer_successes; + dest->optimizer_failure_reason_no_memory += src->optimizer_failure_reason_no_memory; + dest->remove_globals_builtins_changed += src->remove_globals_builtins_changed; + dest->remove_globals_incorrect_keys += src->remove_globals_incorrect_keys; + dest->jit_total_memory_size += src->jit_total_memory_size; + dest->jit_code_size += src->jit_code_size; + dest->jit_trampoline_size += src->jit_trampoline_size; + dest->jit_data_size += src->jit_data_size; + dest->jit_padding_size += src->jit_padding_size; + dest->jit_freed_memory_size += src->jit_freed_memory_size; + + merge_uop_stats_array(dest->opcode, src->opcode); + + for (int i = 0; i < 256; i++) { + dest->unsupported_opcode[i] += src->unsupported_opcode[i]; + } + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + dest->trace_length_hist[i] += src->trace_length_hist[i]; + dest->trace_run_length_hist[i] += src->trace_run_length_hist[i]; + dest->optimized_trace_length_hist[i] += src->optimized_trace_length_hist[i]; + dest->trace_total_memory_hist[i] += src->trace_total_memory_hist[i]; + } + for (int i = 0; i <= PYSTATS_MAX_UOP_ID; i++) { + dest->error_in_opcode[i] += src->error_in_opcode[i]; + } +} + +static void +merge_ft_stats(FTStats *dest, const FTStats *src) +{ + dest->mutex_sleeps = src->mutex_sleeps; + dest->qsbr_polls = src->qsbr_polls; + dest->world_stops = src->world_stops; +} + +static void +merge_rare_event_stats(RareEventStats *dest, const RareEventStats *src) +{ + dest->set_class += src->set_class; + dest->set_bases += src->set_bases; + dest->set_eval_frame_func += src->set_eval_frame_func; + dest->builtin_dict += src->builtin_dict; + dest->func_modification += src->func_modification; + dest->watched_dict_modification += src->watched_dict_modification; + dest->watched_globals_modification += src->watched_globals_modification; +} + +static void +merge_gc_stats_array(GCStats *dest, const GCStats *src) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + dest[i].collections += src[i].collections; + dest[i].object_visits += src[i].object_visits; + dest[i].objects_collected += src[i].objects_collected; + dest[i].objects_transitively_reachable += src[i].objects_transitively_reachable; + dest[i].objects_not_transitively_reachable += src[i].objects_not_transitively_reachable; + } +} + +void +stats_zero_thread(_PyThreadStateImpl *tstate) +{ + // Zero the thread local stat counters + if (tstate->pystats_struct) { + memset(tstate->pystats_struct, 0, sizeof(PyStats)); + } +} + +// merge stats for a single thread into the global structure +void +stats_merge_thread(_PyThreadStateImpl *tstate) +{ + PyStats *src = tstate->pystats_struct; + PyStats *dest = ((PyThreadState *)tstate)->interp->pystats_struct; + + if (src == NULL || dest == NULL) { + return; + } + + // Merge each category of stats using the helper functions. + merge_opcode_stats_array(dest->opcode_stats, src->opcode_stats); + merge_call_stats(&dest->call_stats, &src->call_stats); + merge_object_stats(&dest->object_stats, &src->object_stats); + merge_optimization_stats(&dest->optimization_stats, &src->optimization_stats); + merge_ft_stats(&dest->ft_stats, &src->ft_stats); + merge_rare_event_stats(&dest->rare_event_stats, &src->rare_event_stats); + merge_gc_stats_array(dest->gc_stats, src->gc_stats); +} +#endif // Py_GIL_DISABLED + +// toggle stats collection on or off for all threads +static int +stats_toggle_on_off(PyThreadState *tstate, int on) +{ + bool changed = false; + PyInterpreterState *interp = tstate->interp; + STATS_LOCK(interp); + if (on && interp->pystats_struct == NULL) { + PyStats *s = PyMem_RawCalloc(1, sizeof(PyStats)); + if (s == NULL) { + STATS_UNLOCK(interp); + return -1; + } + FT_ATOMIC_STORE_PTR_RELAXED(interp->pystats_struct, s); + } + if (tstate->interp->pystats_enabled != on) { + FT_ATOMIC_STORE_INT_RELAXED(interp->pystats_enabled, on); + changed = true; + } + STATS_UNLOCK(interp); + if (!changed) { + return 0; + } + _PyEval_StopTheWorld(interp); + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + PyStats *s = NULL; + if (interp->pystats_enabled) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *ts_impl = (_PyThreadStateImpl *)ts; + if (ts_impl->pystats_struct == NULL) { + // first activation for this thread, allocate structure + ts_impl->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + } + s = ts_impl->pystats_struct; +#else + s = ts->interp->pystats_struct; +#endif + } + ts->pystats = s; + } + _PyEval_StartTheWorld(interp); + return 0; +} + +// zero stats for all threads and for the interpreter +static void +stats_zero_all(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return; + } + if (FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct) == NULL) { + return; + } + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + stats_zero_thread((_PyThreadStateImpl *)ts); + } +#endif + if (interp->pystats_struct) { + memset(interp->pystats_struct, 0, sizeof(PyStats)); + } + _PyEval_StartTheWorld(interp); +} + +// merge stats for all threads into the per-interpreter structure +static void +stats_merge_all(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return; + } + if (FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct) == NULL) { + return; + } + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + stats_merge_thread((_PyThreadStateImpl *)ts); + stats_zero_thread((_PyThreadStateImpl *)ts); + } +#endif + _PyEval_StartTheWorld(interp); +} + +int +_Py_StatsOn(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return stats_toggle_on_off(tstate, 1); +} + +void +_Py_StatsOff(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + stats_toggle_on_off(tstate, 0); +} + +void +_Py_StatsClear(void) +{ + stats_zero_all(); +} + +static int +mem_is_zero(unsigned char *ptr, size_t size) +{ + for (size_t i=0; i < size; i++) { + if (*ptr != 0) { + return 0; + } + ptr++; + } + return 1; +} + +int +_Py_PrintSpecializationStats(int to_file) +{ + assert(to_file); + stats_merge_all(); + PyThreadState *tstate = _PyThreadState_GET(); + STATS_LOCK(tstate->interp); + PyStats *stats = tstate->interp->pystats_struct; + if (stats == NULL) { + STATS_UNLOCK(tstate->interp); + return 0; + } +#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) + int is_zero = ( + MEM_IS_ZERO(stats->gc_stats) // is a pointer + && MEM_IS_ZERO(&stats->opcode_stats) + && MEM_IS_ZERO(&stats->call_stats) + && MEM_IS_ZERO(&stats->object_stats) + ); +#undef MEM_IS_ZERO + STATS_UNLOCK(tstate->interp); + if (is_zero) { + // gh-108753: -X pystats command line was used, but then _stats_off() + // and _stats_clear() have been called: in this case, avoid printing + // useless "all zeros" statistics. + return 0; + } + + FILE *out = stderr; + if (to_file) { + /* Write to a file instead of stderr. */ +# ifdef MS_WINDOWS + const char *dirname = "c:\\temp\\py_stats\\"; +# else + const char *dirname = "/tmp/py_stats/"; +# endif + /* Use random 160 bit number as file name, + * to avoid both accidental collisions and + * symlink attacks. */ + unsigned char rand[20]; + char hex_name[41]; + _PyOS_URandomNonblock(rand, 20); + for (int i = 0; i < 20; i++) { + hex_name[2*i] = Py_hexdigits[rand[i]&15]; + hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; + } + hex_name[40] = '\0'; + char buf[64]; + assert(strlen(dirname) + 40 + strlen(".txt") < 64); + sprintf(buf, "%s%s.txt", dirname, hex_name); + FILE *fout = fopen(buf, "w"); + if (fout) { + out = fout; + } + } + else { + fprintf(out, "Specialization stats:\n"); + } + STATS_LOCK(tstate->interp); + print_stats(out, stats); + STATS_UNLOCK(tstate->interp); + if (out != stderr) { + fclose(out); + } + return 1; +} + +PyStatus +_PyStats_InterpInit(PyInterpreterState *interp) +{ + if (interp->config._pystats) { + // start with pystats enabled, can be disabled via sys._stats_off() + // this needs to be set before the first tstate is created + interp->pystats_enabled = 1; + interp->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + if (interp->pystats_struct == NULL) { + return _PyStatus_ERR("out-of-memory while initializing interpreter"); + } + } + return _PyStatus_OK(); +} + +bool +_PyStats_ThreadInit(PyInterpreterState *interp, _PyThreadStateImpl *tstate) +{ +#ifdef Py_GIL_DISABLED + if (FT_ATOMIC_LOAD_INT_RELAXED(interp->pystats_enabled)) { + assert(interp->pystats_struct != NULL); + tstate->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + if (tstate->pystats_struct == NULL) { + return false; + } + } +#endif + return true; +} + +void +_PyStats_ThreadFini(_PyThreadStateImpl *tstate) +{ +#ifdef Py_GIL_DISABLED + STATS_LOCK(((PyThreadState *)tstate)->interp); + stats_merge_thread(tstate); + STATS_UNLOCK(((PyThreadState *)tstate)->interp); + PyMem_RawFree(tstate->pystats_struct); +#endif +} + +void +_PyStats_Attach(_PyThreadStateImpl *tstate_impl) +{ + PyStats *s; + PyThreadState *tstate = (PyThreadState *)tstate_impl; + PyInterpreterState *interp = tstate->interp; + if (FT_ATOMIC_LOAD_INT_RELAXED(interp->pystats_enabled)) { +#ifdef Py_GIL_DISABLED + s = ((_PyThreadStateImpl *)tstate)->pystats_struct; +#else + s = tstate->interp->pystats_struct; +#endif + } + else { + s = NULL; + } + tstate->pystats = s; +} + +void +_PyStats_Detach(_PyThreadStateImpl *tstate_impl) +{ + ((PyThreadState *)tstate_impl)->pystats = NULL; +} + +#endif // Py_STATS diff --git a/Python/qsbr.c b/Python/qsbr.c index c992c285cb1..b2153bf9d67 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -36,6 +36,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_qsbr.h" #include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_stats.h" // FT_STAT_QSBR_POLL_INC() // Starting size of the array of qsbr thread states @@ -158,7 +159,7 @@ _Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) if (_Py_qbsr_goal_reached(qsbr, goal)) { return true; } - + FT_STAT_QSBR_POLL_INC(); uint64_t rd_seq = qsbr_poll_scan(qsbr->shared); return QSBR_LEQ(goal, rd_seq); } diff --git a/Python/specialize.c b/Python/specialize.c index a1c5dedd615..2193596a331 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -22,437 +22,23 @@ #include // rand() -extern const char *_PyUOpName(int index); - /* For guidance on adding or extending families of instructions see * InternalDocs/interpreter.md `Specialization` section. */ -#ifdef Py_STATS -GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; -static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; -PyStats *_Py_stats = NULL; - -#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID -#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" -#endif - -#define ADD_STAT_TO_DICT(res, field) \ - do { \ - PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ - if (val == NULL) { \ - Py_DECREF(res); \ - return NULL; \ - } \ - if (PyDict_SetItemString(res, #field, val) == -1) { \ - Py_DECREF(res); \ - Py_DECREF(val); \ - return NULL; \ - } \ - Py_DECREF(val); \ - } while(0); - -static PyObject* -stats_to_dict(SpecializationStats *stats) -{ - PyObject *res = PyDict_New(); - if (res == NULL) { - return NULL; - } - ADD_STAT_TO_DICT(res, success); - ADD_STAT_TO_DICT(res, failure); - ADD_STAT_TO_DICT(res, hit); - ADD_STAT_TO_DICT(res, deferred); - ADD_STAT_TO_DICT(res, miss); - ADD_STAT_TO_DICT(res, deopt); - PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); - if (failure_kinds == NULL) { - Py_DECREF(res); - return NULL; - } - for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { - PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); - if (stat == NULL) { - Py_DECREF(res); - Py_DECREF(failure_kinds); - return NULL; - } - PyTuple_SET_ITEM(failure_kinds, i, stat); - } - if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { - Py_DECREF(res); - Py_DECREF(failure_kinds); - return NULL; - } - Py_DECREF(failure_kinds); - return res; -} -#undef ADD_STAT_TO_DICT - -static int -add_stat_dict( - PyObject *res, - int opcode, - const char *name) { - - SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization; - PyObject *d = stats_to_dict(stats); - if (d == NULL) { - return -1; - } - int err = PyDict_SetItemString(res, name, d); - Py_DECREF(d); - return err; -} - -PyObject* -_Py_GetSpecializationStats(void) { - PyObject *stats = PyDict_New(); - if (stats == NULL) { - return NULL; - } - int err = 0; - err += add_stat_dict(stats, CONTAINS_OP, "contains_op"); - err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr"); - err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); - err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); - err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); - err += add_stat_dict(stats, STORE_ATTR, "store_attr"); - err += add_stat_dict(stats, JUMP_BACKWARD, "jump_backward"); - err += add_stat_dict(stats, CALL, "call"); - err += add_stat_dict(stats, CALL_KW, "call_kw"); - err += add_stat_dict(stats, BINARY_OP, "binary_op"); - err += add_stat_dict(stats, COMPARE_OP, "compare_op"); - err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence"); - err += add_stat_dict(stats, FOR_ITER, "for_iter"); - err += add_stat_dict(stats, TO_BOOL, "to_bool"); - err += add_stat_dict(stats, SEND, "send"); - if (err < 0) { - Py_DECREF(stats); - return NULL; - } - return stats; -} - - -#define PRINT_STAT(i, field) \ - if (stats[i].field) { \ - fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \ - } - -static void -print_spec_stats(FILE *out, OpcodeStats *stats) -{ - /* Mark some opcodes as specializable for stats, - * even though we don't specialize them yet. */ - fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); - fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); - fprintf(out, "opcode[GET_ITER].specializable : 1\n"); - for (int i = 0; i < 256; i++) { - if (_PyOpcode_Caches[i]) { - /* Ignore jumps as they cannot be specialized */ - switch (i) { - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case JUMP_BACKWARD: - break; - default: - fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); - } - } - PRINT_STAT(i, specialization.success); - PRINT_STAT(i, specialization.failure); - PRINT_STAT(i, specialization.hit); - PRINT_STAT(i, specialization.deferred); - PRINT_STAT(i, specialization.miss); - PRINT_STAT(i, specialization.deopt); - PRINT_STAT(i, execution_count); - for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { - uint64_t val = stats[i].specialization.failure_kinds[j]; - if (val) { - fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" - PRIu64 "\n", _PyOpcode_OpName[i], j, val); - } - } - for (int j = 0; j < 256; j++) { - if (stats[i].pair_count[j]) { - fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", - _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); - } - } - } -} -#undef PRINT_STAT - - -static void -print_call_stats(FILE *out, CallStats *stats) -{ - fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); - fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); - fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); - fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); - for (int i = 0; i < EVAL_CALL_KINDS; i++) { - fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); - } -} - -static void -print_object_stats(FILE *out, ObjectStats *stats) -{ - fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); - fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); - fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); - fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); - fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); - fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); - fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); - fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); - fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); - fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); - fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); - fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); - fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); - fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); - fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); - fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); - fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); - fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); - fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); - fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); - fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); - fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); - fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); -} - -static void -print_gc_stats(FILE *out, GCStats *stats) -{ - for (int i = 0; i < NUM_GENERATIONS; i++) { - fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); - fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); - fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); - fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); - fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); - } -} - -#ifdef _Py_TIER2 -static void -print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) -{ - for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { - fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); - } -} - -static void -print_optimization_stats(FILE *out, OptimizationStats *stats) -{ - fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); - fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); - fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); - fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); - fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); - fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); - fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); - fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); - fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); - fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); - fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); - fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); - fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); - - print_histogram(out, "Trace length", stats->trace_length_hist); - print_histogram(out, "Trace run length", stats->trace_run_length_hist); - print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); - - fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); - fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); - fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", - stats->optimizer_failure_reason_no_memory); - fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); - fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { - if (stats->opcode[i].execution_count) { - fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); - } - if (stats->opcode[i].miss) { - fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); - } - } - for (int i = 0; i < 256; i++) { - if (stats->unsupported_opcode[i]) { - fprintf( - out, - "unsupported_opcode[%s].count : %" PRIu64 "\n", - _PyOpcode_OpName[i], - stats->unsupported_opcode[i] - ); - } - } - - for (int i = 1; i <= MAX_UOP_ID; i++){ - for (int j = 1; j <= MAX_UOP_ID; j++) { - if (stats->opcode[i].pair_count[j]) { - fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", - _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); - } - } - } - for (int i = 0; i < MAX_UOP_ID; i++) { - if (stats->error_in_opcode[i]) { - fprintf( - out, - "error_in_opcode[%s].count : %" PRIu64 "\n", - _PyUOpName(i), - stats->error_in_opcode[i] - ); - } - } - fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size); - fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size); - fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size); - fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size); - fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size); - fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size); - - print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist); -} -#endif - -static void -print_rare_event_stats(FILE *out, RareEventStats *stats) -{ - fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); - fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); - fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); - fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); - fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); - fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); - fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); -} - -static void -print_stats(FILE *out, PyStats *stats) -{ - print_spec_stats(out, stats->opcode_stats); - print_call_stats(out, &stats->call_stats); - print_object_stats(out, &stats->object_stats); - print_gc_stats(out, stats->gc_stats); -#ifdef _Py_TIER2 - print_optimization_stats(out, &stats->optimization_stats); -#endif - print_rare_event_stats(out, &stats->rare_event_stats); -} - -void -_Py_StatsOn(void) -{ - _Py_stats = &_Py_stats_struct; -} - -void -_Py_StatsOff(void) -{ - _Py_stats = NULL; -} - -void -_Py_StatsClear(void) -{ - memset(&_py_gc_stats, 0, sizeof(_py_gc_stats)); - memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct)); - _Py_stats_struct.gc_stats = _py_gc_stats; -} - -static int -mem_is_zero(unsigned char *ptr, size_t size) -{ - for (size_t i=0; i < size; i++) { - if (*ptr != 0) { - return 0; - } - ptr++; - } - return 1; -} - -int -_Py_PrintSpecializationStats(int to_file) -{ - PyStats *stats = &_Py_stats_struct; -#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) - int is_zero = ( - MEM_IS_ZERO(stats->gc_stats) // is a pointer - && MEM_IS_ZERO(&stats->opcode_stats) - && MEM_IS_ZERO(&stats->call_stats) - && MEM_IS_ZERO(&stats->object_stats) - ); -#undef MEM_IS_ZERO - if (is_zero) { - // gh-108753: -X pystats command line was used, but then _stats_off() - // and _stats_clear() have been called: in this case, avoid printing - // useless "all zeros" statistics. - return 0; - } - - FILE *out = stderr; - if (to_file) { - /* Write to a file instead of stderr. */ -# ifdef MS_WINDOWS - const char *dirname = "c:\\temp\\py_stats\\"; -# else - const char *dirname = "/tmp/py_stats/"; -# endif - /* Use random 160 bit number as file name, - * to avoid both accidental collisions and - * symlink attacks. */ - unsigned char rand[20]; - char hex_name[41]; - _PyOS_URandomNonblock(rand, 20); - for (int i = 0; i < 20; i++) { - hex_name[2*i] = Py_hexdigits[rand[i]&15]; - hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; - } - hex_name[40] = '\0'; - char buf[64]; - assert(strlen(dirname) + 40 + strlen(".txt") < 64); - sprintf(buf, "%s%s.txt", dirname, hex_name); - FILE *fout = fopen(buf, "w"); - if (fout) { - out = fout; - } - } - else { - fprintf(out, "Specialization stats:\n"); - } - print_stats(out, stats); - if (out != stderr) { - fclose(out); - } - return 1; -} - +#if Py_STATS #define SPECIALIZATION_FAIL(opcode, kind) \ do { \ - if (_Py_stats) { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ int _kind = (kind); \ assert(_kind < SPECIALIZATION_FAILURE_KINDS); \ - _Py_stats->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \ + s->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \ } \ } while (0) - -#endif // Py_STATS - - -#ifndef SPECIALIZATION_FAIL +#else # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) -#endif +#endif // Py_STATS // Initialize warmup counters and optimize instructions. This cannot fail. void diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 59baca26793..86dd1395cae 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2281,7 +2281,9 @@ static PyObject * sys__stats_on_impl(PyObject *module) /*[clinic end generated code: output=aca53eafcbb4d9fe input=43b5bfe145299e55]*/ { - _Py_StatsOn(); + if (_Py_StatsOn() < 0) { + return NULL; + } Py_RETURN_NONE; } From 57f4d09a6fcfec229603f644c4254abdded00429 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 11:46:53 -0800 Subject: [PATCH 004/417] JIT: Fix compiler warning from visibility attribute in typedef (#139981) --- Tools/jit/trampoline.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c index 79d6af97961..fdc63b7fc40 100644 --- a/Tools/jit/trampoline.c +++ b/Tools/jit/trampoline.c @@ -10,7 +10,7 @@ _Py_CODEUNIT * _JIT_ENTRY( _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ) { - typedef DECLARE_TARGET((*jit_func)); - jit_func jitted = (jit_func)exec->jit_code; + // Note that this is *not* a tail call + jit_func_preserve_none jitted = (jit_func_preserve_none)exec->jit_code; return jitted(frame, stack_pointer, tstate); } From 3f2b83e9595ff2436a646e6bbd335198c4bc06db Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 3 Nov 2025 21:04:46 +0100 Subject: [PATCH 005/417] Fix minor typos and wording in C API docs (#140955) --- Doc/c-api/cell.rst | 2 +- Doc/c-api/complex.rst | 2 +- Doc/c-api/datetime.rst | 2 +- Doc/c-api/descriptor.rst | 2 +- Doc/c-api/init_config.rst | 2 +- Doc/c-api/mapping.rst | 4 ++-- Doc/c-api/marshal.rst | 2 +- Doc/c-api/memory.rst | 2 +- Doc/c-api/module.rst | 2 +- Doc/c-api/monitoring.rst | 2 +- Doc/c-api/object.rst | 2 +- Doc/c-api/stable.rst | 2 +- Doc/c-api/tuple.rst | 2 +- Doc/c-api/veryhigh.rst | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c370..2501ed9580d 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index d135637a741..629312bd771 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -82,7 +82,7 @@ Complex Number Objects .. c:type:: Py_complex - This C structure defines export format for a Python complex + This C structure defines an export format for a Python complex number object. .. c:member:: double real diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c7..f311aad5f15 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -46,7 +46,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f0..ff0df575279 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -32,7 +32,7 @@ found in the dictionary of type objects. .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index b20495e672d..c345029e4ac 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -102,7 +102,7 @@ Error Handling * Set *\*err_msg* and return ``1`` if an error is set. * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - An error message is an UTF-8 encoded string. + An error message is a UTF-8 encoded string. If *config* has an exit code, format the exit code as an error message. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955..2476ebb9b69 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 61218a1bf6f..668a163b2df 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -82,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index df1bb0ce370..23958980102 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -102,7 +102,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6626f628fcc..ed2a7663375 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -103,7 +103,7 @@ Module Objects created, or ``NULL`` if the module wasn't created from a definition. On error, return ``NULL`` with an exception set. - Use :c:func:`PyErr_Occurred` to tell this case apart from a mising + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing :c:type:`!PyModuleDef`. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 7926148302a..b0227c2f4fa 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -136,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 8629b768a29..96353266ac7 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index b08a7bf1b2f..f5e6b7ad157 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -279,7 +279,7 @@ The full API is described below for advanced use cases. .. c:member:: uint8_t abiinfo_minor_version - The major version of :c:struct:`PyABIInfo`. + The minor version of :c:struct:`PyABIInfo`. Must be set to ``0``; larger values are reserved for backwards-compatible future versions of :c:struct:`!PyABIInfo`. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 14a7c05efea..d0add48d7e8 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -61,7 +61,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index ee0595a9e08..0b2b55b6387 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -140,7 +140,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 From 947bb4642c612b66de7cae815aaf9c570a93882a Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Tue, 4 Nov 2025 04:37:29 +0800 Subject: [PATCH 006/417] gh-140889: Bump tailcall and JIT CI to llvm 20 (#140963) --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 40d8b74e982..69d900091a3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 19 + - 20 steps: - uses: actions/checkout@v4 with: From 08115d241a724a4769599993f654f77abcdebf5a Mon Sep 17 00:00:00 2001 From: commitWithTisha Date: Tue, 4 Nov 2025 03:23:49 -0600 Subject: [PATCH 007/417] Fix minor typo: 'web site' -> 'website' (GH-140561) --- Doc/library/http.cookiejar.rst | 2 +- Doc/library/urllib.robotparser.rst | 2 +- Doc/tutorial/index.rst | 2 +- Doc/tutorial/whatnow.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 251aea891c3..fcb0069b760 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -12,7 +12,7 @@ -------------- The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP -cookies. It is useful for accessing web sites that require small pieces of data +cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 016fcdc75da..674f646c633 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -19,7 +19,7 @@ This module provides a single class, :class:`RobotFileParser`, which answers questions about whether or not a particular user agent can fetch a URL on the -web site that published the :file:`robots.txt` file. For more details on the +website that published the :file:`robots.txt` file. For more details on the structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index d0bf77dc40d..20fe161be4a 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -15,7 +15,7 @@ together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, +in source or binary form for all major platforms from the Python website, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index dbe2d7fc099..359cf80a7b2 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -30,7 +30,7 @@ the set are: More Python resources: -* https://www.python.org: The major Python web site. It contains code, +* https://www.python.org: The major Python website. It contains code, documentation, and pointers to Python-related pages around the web. * https://docs.python.org: Fast access to Python's documentation. From a84181c31bfc45a1d6bcb1296bd298ad612c54d0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Nov 2025 11:48:28 +0100 Subject: [PATCH 008/417] gh-140815: Fix faulthandler for invalid/freed frame (#140921) faulthandler now detects if a frame or a code object is invalid or freed. Add helper functions: * _PyCode_SafeAddr2Line() * _PyFrame_SafeGetCode() * _PyFrame_SafeGetLasti() _PyMem_IsPtrFreed() now detects pointers in [-0xff, 0xff] range as freed. --- Include/internal/pycore_code.h | 9 ++- Include/internal/pycore_interpframe.h | 55 ++++++++++++++++ Include/internal/pycore_pymem.h | 10 +-- ...-11-02-19-23-32.gh-issue-140815.McEG-T.rst | 2 + Objects/codeobject.c | 23 ++++++- Python/traceback.c | 62 ++++++++++++------- 6 files changed, 128 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 2d7d81d491c..9748e036bf2 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -274,8 +274,13 @@ extern void _PyLineTable_InitAddressRange( /** API for traversing the line number table. */ PyAPI_FUNC(int) _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); -// This is used in dump_frame() in traceback.c without an attached tstate. -extern int _PyCode_Addr2LineNoTstate(PyCodeObject *co, int addr); + +// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid +// and can be called without an attached tstate. Used by dump_frame() in +// Python/traceback.c. The function uses heuristics to detect freed memory, +// it's not 100% reliable. +extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); + /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 2ee3696317c..8949d6cc2fc 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -24,6 +24,36 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +// Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or +// freed. Used by dump_frame() in Python/traceback.c. The function uses +// heuristics to detect freed memory, it's not 100% reliable. +static inline PyCodeObject* +_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +{ + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (f->f_globals == NULL || f->f_builtins == NULL) { + return NULL; + } + + if (PyStackRef_IsNull(f->f_executable)) { + return NULL; + } + void *ptr; + memcpy(&ptr, &f->f_executable, sizeof(f->f_executable)); + if (_PyMem_IsPtrFreed(ptr)) { + return NULL; + } + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (_PyObject_IsFreed(executable)) { + return NULL; + } + if (!PyCode_Check(executable)) { + return NULL; + } + return (PyCodeObject *)executable; +} + static inline _Py_CODEUNIT * _PyFrame_GetBytecode(_PyInterpreterFrame *f) { @@ -37,6 +67,31 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) #endif } +// Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the +// frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The +// function uses heuristics to detect freed memory, it's not 100% reliable. +static inline int +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +{ + // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() + // with _PyFrame_SafeGetCode(). + PyCodeObject *co = _PyFrame_SafeGetCode(f); + if (co == NULL) { + return -1; + } + + _Py_CODEUNIT *bytecode; +#ifdef Py_GIL_DISABLED + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + bytecode = (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + bytecode = _PyCode_CODE(co); +#endif + + return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index f3f2ae0a140..ed943b51056 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -54,15 +54,17 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == 0 + return (value <= 0xff // NULL, 0x1, 0x2, ..., 0xff || value == (uintptr_t)0xCDCDCDCDCDCDCDCD || value == (uintptr_t)0xDDDDDDDDDDDDDDDD - || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD + || value >= (uintptr_t)0xFFFFFFFFFFFFFF00); // -0xff, ..., -2, -1 #elif SIZEOF_VOID_P == 4 - return (value == 0 + return (value <= 0xff || value == (uintptr_t)0xCDCDCDCD || value == (uintptr_t)0xDDDDDDDD - || value == (uintptr_t)0xFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFD + || value >= (uintptr_t)0xFFFFFF00); #else # error "unknown pointer size" #endif diff --git a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst new file mode 100644 index 00000000000..18c4d3836ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst @@ -0,0 +1,2 @@ +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d264a6e346..fc3f5d9dde0 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1005,8 +1005,8 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) * source location tracking (co_lines/co_positions) ******************/ -int -_PyCode_Addr2LineNoTstate(PyCodeObject *co, int addrq) +static int +_PyCode_Addr2Line(PyCodeObject *co, int addrq) { if (addrq < 0) { return co->co_firstlineno; @@ -1020,12 +1020,29 @@ _PyCode_Addr2LineNoTstate(PyCodeObject *co, int addrq) return _PyCode_CheckLineNumber(addrq, &bounds); } +int +_PyCode_SafeAddr2Line(PyCodeObject *co, int addrq) +{ + if (addrq < 0) { + return co->co_firstlineno; + } + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + } + if (!(addrq >= 0 && addrq < _PyCode_NBYTES(co))) { + return -1; + } + PyCodeAddressRange bounds; + _PyCode_InitAddressRange(co, &bounds); + return _PyCode_CheckLineNumber(addrq, &bounds); +} + int PyCode_Addr2Line(PyCodeObject *co, int addrq) { int lineno; Py_BEGIN_CRITICAL_SECTION(co); - lineno = _PyCode_Addr2LineNoTstate(co, addrq); + lineno = _PyCode_Addr2Line(co, addrq); Py_END_CRITICAL_SECTION(); return lineno; } diff --git a/Python/traceback.c b/Python/traceback.c index ef67368550a..521d6322a5c 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1028,14 +1028,24 @@ _Py_DumpWideString(int fd, wchar_t *str) /* Write a frame into the file fd: "File "xxx", line xxx in xxx". - This function is signal safe. */ + This function is signal safe. -static void + Return 0 on success. Return -1 if the frame is invalid. */ + +static int dump_frame(int fd, _PyInterpreterFrame *frame) { - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Ignore trampoline frame */ + return 0; + } - PyCodeObject *code =_PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_SafeGetCode(frame); + if (code == NULL) { + return -1; + } + + int res = 0; PUTS(fd, " File "); if (code->co_filename != NULL && PyUnicode_Check(code->co_filename)) @@ -1043,29 +1053,36 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "\""); _Py_DumpASCII(fd, code->co_filename); PUTS(fd, "\""); - } else { - PUTS(fd, "???"); } - int lasti = PyUnstable_InterpreterFrame_GetLasti(frame); - int lineno = _PyCode_Addr2LineNoTstate(code, lasti); + else { + PUTS(fd, "???"); + res = -1; + } + PUTS(fd, ", line "); + int lasti = _PyFrame_SafeGetLasti(frame); + int lineno = -1; + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, " in "); - if (code->co_name != NULL - && PyUnicode_Check(code->co_name)) { + PUTS(fd, " in "); + if (code->co_name != NULL && PyUnicode_Check(code->co_name)) { _Py_DumpASCII(fd, code->co_name); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, "\n"); + return res; } static int @@ -1108,17 +1125,6 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Trampoline frame */ - frame = frame->previous; - if (frame == NULL) { - break; - } - - /* Can't have more than one shim frame in a row */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - } - if (MAX_FRAME_DEPTH <= depth) { if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); @@ -1128,7 +1134,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) break; } - dump_frame(fd, frame); + if (_PyMem_IsPtrFreed(frame)) { + PUTS(fd, " \n"); + break; + } + if (dump_frame(fd, frame) < 0) { + PUTS(fd, " \n"); + break; + } + frame = frame->previous; if (frame == NULL) { break; From fa9c3eefd475f0647a69bf3f49db8100848fb6a9 Mon Sep 17 00:00:00 2001 From: Abhishek Tiwari Date: Tue, 4 Nov 2025 16:24:28 +0530 Subject: [PATCH 009/417] gh-140797: Forbid capturing groups in re.Scanner lexicon patterns (GH-140944) --- Lib/re/__init__.py | 5 ++++- Lib/test/test_re.py | 18 ++++++++++++++++++ ...5-11-03-16-23-54.gh-issue-140797.DuFEeR.rst | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index a5316391297..ecec16e9005 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -397,9 +397,12 @@ def __init__(self, lexicon, flags=0): s = _parser.State() s.flags = flags for phrase, action in lexicon: + sub_pattern = _parser.parse(phrase, flags) + if sub_pattern.state.groups != 1: + raise ValueError("Cannot use capturing groups in re.Scanner") gid = s.opengroup() p.append(_parser.SubPattern(s, [ - (SUBPATTERN, (gid, 0, 0, _parser.parse(phrase, flags))), + (SUBPATTERN, (gid, 0, 0, sub_pattern)), ])) s.closegroup(gid, p[-1]) p = _parser.SubPattern(s, [(BRANCH, (None, p))]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 5fc95087f2b..9f6f04bf6b8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1639,6 +1639,24 @@ def s_int(scanner, token): return int(token) (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5, 'op+', 'bar'], '')) + def test_bug_gh140797(self): + # gh140797: Capturing groups are not allowed in re.Scanner + + msg = r"Cannot use capturing groups in re\.Scanner" + # Capturing group throws an error + with self.assertRaisesRegex(ValueError, msg): + Scanner([("(a)b", None)]) + + # Named Group + with self.assertRaisesRegex(ValueError, msg): + Scanner([("(?Pa)", None)]) + + # Non-capturing groups should pass normally + s = Scanner([("(?:a)b", lambda scanner, token: token)]) + result, rem = s.scan("ab") + self.assertEqual(result,['ab']) + self.assertEqual(rem,'') + def test_bug_448951(self): # bug 448951 (similar to 429357, but with single char match) # (Also test greedy matches.) diff --git a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst new file mode 100644 index 00000000000..493b740261e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst @@ -0,0 +1,2 @@ +The undocumented :class:`!re.Scanner` class now forbids regular expressions containing capturing groups in its lexicon patterns. Patterns using capturing groups could +previously lead to crashes with segmentation fault. Use non-capturing groups (?:...) instead. From 1326d2a808245e5f2de9e515460bab30556e8f05 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 4 Nov 2025 17:49:44 +0200 Subject: [PATCH 010/417] gh-140979: Fix off-by-one error in the RE code validator (GH-140984) It was too lenient and allowed MARK opcodes with too large value. --- Modules/_sre/sre.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index fdf00e6499c..4e97101b699 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1946,7 +1946,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) sre_match() code is robust even if they don't, and the worst you can get is nonsensical match results. */ GET_ARG; - if (arg > 2 * (size_t)groups + 1) { + if (arg >= 2 * (size_t)groups) { VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); FAIL; } From 40096da95a592ac0b2ad6aa9c731631784c3b393 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 4 Nov 2025 08:31:35 -0800 Subject: [PATCH 011/417] GH-139946: Colorize error and warning messages in argparse (#140695) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/_colorize.py | 3 ++ Lib/argparse.py | 22 ++++++++-- Lib/test/test_argparse.py | 41 +++++++++++++++++++ Lib/test/test_clinic.py | 2 + Lib/test/test_gzip.py | 3 +- Lib/test/test_uuid.py | 4 +- Lib/test/test_webbrowser.py | 2 + ...-10-28-02-46-56.gh-issue-139946.aN3_uY.rst | 1 + 8 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 63e951d6488..57b712bc068 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -170,6 +170,9 @@ class Argparse(ThemeSection): label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN reset: str = ANSIColors.RESET + error: str = ANSIColors.BOLD_MAGENTA + warning: str = ANSIColors.BOLD_YELLOW + message: str = ANSIColors.MAGENTA @dataclass(frozen=True, kw_only=True) diff --git a/Lib/argparse.py b/Lib/argparse.py index 1f4413a9897..6b79747572f 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2749,6 +2749,14 @@ def _print_message(self, message, file=None): except (AttributeError, OSError): pass + def _get_theme(self, file=None): + from _colorize import can_colorize, get_theme + + if self.color and can_colorize(file=file): + return get_theme(force_color=True).argparse + else: + return get_theme(force_no_color=True).argparse + # =============== # Exiting methods # =============== @@ -2768,13 +2776,21 @@ def error(self, message): should either exit or raise an exception. """ self.print_usage(_sys.stderr) + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: error: %(message)s\n') + fmt = fmt.replace('error: %(message)s', + f'{theme.error}error:{theme.reset} {theme.message}%(message)s{theme.reset}') + args = {'prog': self.prog, 'message': message} - self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + self.exit(2, fmt % args) def _warning(self, message): + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: warning: %(message)s\n') + fmt = fmt.replace('warning: %(message)s', + f'{theme.warning}warning:{theme.reset} {theme.message}%(message)s{theme.reset}') args = {'prog': self.prog, 'message': message} - self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) - + self._print_message(fmt % args, _sys.stderr) def __getattr__(name): if name == "__version__": diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index d6c9c1ef2c8..3a8be68a546 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2283,6 +2283,7 @@ class TestNegativeNumber(ParserTestCase): ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), ] +@force_not_colorized_test_class class TestArgumentAndSubparserSuggestions(TestCase): """Test error handling and suggestion when a user makes a typo""" @@ -6147,6 +6148,7 @@ def spam(string_to_convert): # Check that deprecated arguments output warning # ============================================== +@force_not_colorized_test_class class TestDeprecatedArguments(TestCase): def test_deprecated_option(self): @@ -7370,6 +7372,45 @@ def test_subparser_prog_is_stored_without_color(self): help_text = demo_parser.format_help() self.assertNotIn('\x1b[', help_text) + def test_error_and_warning_keywords_colorized(self): + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('foo') + + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + parser.parse_args([]) + + err = stderr.getvalue() + error_color = self.theme.error + reset = self.theme.reset + self.assertIn(f'{error_color}error:{reset}', err) + + with captured_stderr() as stderr: + parser._warning('test warning') + + warn = stderr.getvalue() + warning_color = self.theme.warning + self.assertIn(f'{warning_color}warning:{reset}', warn) + + def test_error_and_warning_not_colorized_when_disabled(self): + parser = argparse.ArgumentParser(prog='PROG', color=False) + parser.add_argument('foo') + + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + parser.parse_args([]) + + err = stderr.getvalue() + self.assertNotIn('\x1b[', err) + self.assertIn('error:', err) + + with captured_stderr() as stderr: + parser._warning('test warning') + + warn = stderr.getvalue() + self.assertNotIn('\x1b[', warn) + self.assertIn('warning:', warn) + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index e0dbb062eb0..e71f9fc181b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -4,6 +4,7 @@ from functools import partial from test import support, test_tools +from test.support import force_not_colorized_test_class from test.support import os_helper from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent @@ -2758,6 +2759,7 @@ def test_allow_negative_accepted_by_py_ssize_t_converter_only(self): with self.assertRaisesRegex((AssertionError, TypeError), errmsg): self.parse_function(block) +@force_not_colorized_test_class class ClinicExternalTest(TestCase): maxDiff = None diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index f14a882d386..442d30fc970 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -11,7 +11,7 @@ import unittest from subprocess import PIPE, Popen from test.support import catch_unraisable_exception -from test.support import import_helper +from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper from test.support import _4G, bigmemtest, requires_subprocess from test.support.script_helper import assert_python_ok, assert_python_failure @@ -1057,6 +1057,7 @@ def wrapper(*args, **kwargs): return decorator +@force_not_colorized_test_class class TestCommandLine(unittest.TestCase): data = b'This is a simple test with gzip' diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 33045a78721..5f9ab048cde 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -13,7 +13,7 @@ from unittest import mock from test import support -from test.support import import_helper, warnings_helper +from test.support import force_not_colorized_test_class, import_helper, warnings_helper from test.support.script_helper import assert_python_ok py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) @@ -1250,10 +1250,12 @@ def test_cli_uuid8(self): self.do_test_standalone_uuid(8) +@force_not_colorized_test_class class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid +@force_not_colorized_test_class @unittest.skipUnless(c_uuid, 'requires the C _uuid module') class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 6b577ae100e..20d347168b3 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -7,6 +7,7 @@ import unittest import webbrowser from test import support +from test.support import force_not_colorized_test_class from test.support import import_helper from test.support import is_apple_mobile from test.support import os_helper @@ -503,6 +504,7 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) +@force_not_colorized_test_class class CliTest(unittest.TestCase): def test_parse_args(self): for command, url, new_win in [ diff --git a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst new file mode 100644 index 00000000000..4c68d4cd94b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst @@ -0,0 +1 @@ +Error and warning keywords in ``argparse.ArgumentParser`` messages are now colorized when color output is enabled, fixing a visual inconsistency in which they remained plain text while other output was colorized. From 8a7dbb7a68b5da1f3f1805f564c028f1eea4ebc3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 4 Nov 2025 10:28:17 -0800 Subject: [PATCH 012/417] Document that returning `sys.monitoring.DISABLE` in response to a global event raises `ValueError` (#140726) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/sys.monitoring.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0f986aa580b..303655fb128 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -216,14 +216,17 @@ by another event: The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events are controlled by the :monitoring-event:`CALL` event. -:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the -corresponding :monitoring-event:`CALL` event is being monitored. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be +seen if the corresponding :monitoring-event:`CALL` event is being monitored. + + +.. _monitoring-event-global: Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled. +program and cannot be individually disabled via :data:`DISABLE`. The other events that can be monitored are: @@ -289,12 +292,13 @@ in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for *code* + Returns all the :ref:`local events ` for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for *code* which are set in *event_set*. - Raises a :exc:`ValueError` if *tool_id* is not in use. + Activates all the :ref:`local events ` for *code* + which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not + in use. Disabling events @@ -305,15 +309,21 @@ Disabling events A special value that can be returned from a callback function to disable events for the current code location. -Local events can be disabled for a specific code location by returning -:data:`sys.monitoring.DISABLE` from a callback function. This does not change -which events are set, or any other code locations for the same event. +:ref:`Local events ` can be disabled for a specific code +location by returning :data:`sys.monitoring.DISABLE` from a callback function. +This does not change which events are set, or any other code locations for the +same event. Disabling events for specific locations is very important for high performance monitoring. For example, a program can be run under a debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +If :data:`DISABLE` is returned by a callback for a +:ref:`global event `, :exc:`ValueError` will be raised +by the interpreter in a non-specific location (that is, no traceback will be +provided). + .. function:: restart_events() -> None Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` From 66c86c65633047c0faffba85ce6b0b3a82373657 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Tue, 4 Nov 2025 18:29:44 +0000 Subject: [PATCH 013/417] gh-134817: Restore accidentally deleted line in documentation. (GH-141013) --- Doc/library/logging.handlers.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index d74ef73ee28..c9cfbdb4126 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -463,6 +463,7 @@ timed intervals. .. method:: getFilesToDelete() Returns a list of filenames which should be deleted as part of rollover. These + are the absolute paths of the oldest backup log files written by the handler. .. method:: shouldRollover(record) From 97d8dda980fcddf88b782be343118257f483a864 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Tue, 4 Nov 2025 14:29:13 -0500 Subject: [PATCH 014/417] Docs: Fix typo in `email.headerregistry.rst` (#140965) Fix missing 'Header' suffix on header class name in `email.headerregistry.rst` --- Doc/library/email.headerregistry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 7f8044932fa..ff8b601fe3d 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -294,7 +294,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. ``inline`` and ``attachment`` are the only valid values in common use. -.. class:: ContentTransferEncoding +.. class:: ContentTransferEncodingHeader Handles the :mailheader:`Content-Transfer-Encoding` header. From ce1bb85d286130f44b7e874430b0b12990d61dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=81ajszczak?= Date: Tue, 4 Nov 2025 20:46:07 +0100 Subject: [PATCH 015/417] gh-139434: Update selected RFC 2822 references to RFC 5322 (#139435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update selected RFC 2822 references to RFC 5322 RFC 2822 was obsoleted by RFC 5322 in 2008. This updates references to use the current standard in documentation, docstrings, and comments. It preserves RFC 2822 references in legacy API components to maintain their historical context. RFC 822 → RFC 2822 → RFC 5322 progression is explained where relevant. In some places specific sections of RFC are referenced where it seems helpful. Scout rule was applied in some places and RFC mentions format was normalized in doc strings and comments. --- Doc/library/http.client.rst | 2 +- Doc/library/http.server.rst | 2 +- Doc/library/mailbox.rst | 2 +- Doc/library/time.rst | 5 ++-- Doc/tutorial/stdlib.rst | 2 +- Lib/email/_parseaddr.py | 13 +++++++---- Lib/email/_policybase.py | 2 +- Lib/email/feedparser.py | 4 ++-- Lib/email/generator.py | 2 +- Lib/email/message.py | 2 +- Lib/email/parser.py | 10 ++++---- Lib/http/client.py | 4 ++-- Lib/smtplib.py | 6 ++--- Lib/test/test_email/data/msg_35.txt | 2 +- Lib/test/test_email/test_email.py | 36 ++++++++++++++--------------- 15 files changed, 49 insertions(+), 45 deletions(-) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 589152f2968..7c258b324d9 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -133,7 +133,7 @@ This module provides the following function: Parse the headers from a file pointer *fp* representing a HTTP request/response. The file has to be a :class:`~io.BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. + (i.e. not text) and must provide a valid :rfc:`5322` style header. This function returns an instance of :class:`http.client.HTTPMessage` that holds the header fields, but no payload diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 063344e0284..58f09634f95 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -154,7 +154,7 @@ instantiation, of which this module provides three different variants: variable. This instance parses and manages the headers in the HTTP request. The :func:`~http.client.parse_headers` function from :mod:`http.client` is used to parse the headers and it requires that the - HTTP request provide a valid :rfc:`2822` style header. + HTTP request provide a valid :rfc:`5322` style header. .. attribute:: rfile diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index e8a96f29ea1..62e289573c0 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -917,7 +917,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read + or a file, it should contain an :rfc:`5322`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files are accepted for backward compatibility. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 350ffade7af..69e6433e898 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -584,7 +584,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: + :rfc:`5322` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -1066,4 +1066,5 @@ Timezone Constants strict reading of the original 1982 :rfc:`822` standard calls for a two-digit year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. + been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`, + with :rfc:`5322` continuing this requirement. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index d83ecca270b..49a3e370a4c 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -335,7 +335,7 @@ sophisticated and robust capabilities of its larger packages. For example: names, no direct knowledge or handling of XML is needed. * The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and + MIME and other :rfc:`5322`-based message documents. Unlike :mod:`smtplib` and :mod:`poplib` which actually send and receive messages, the email package has a complete toolset for building or decoding complex message structures (including attachments) and for implementing internet encoding and header diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 84917038874..6a7c5fa06d2 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -146,8 +146,9 @@ def _parsedate_tz(data): return None # Check for a yy specified in two-digit format, then convert it to the # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for + # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) already + # mandated a 4-digit yy, and RFC 5322 (which obsoletes RFC 2822) continues + # this requirement. For more information, see the documentation for # the time module. if yy < 100: # The year is between 1969 and 1999 (inclusive). @@ -233,9 +234,11 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. + # Note that RFC 2822 section 4.1 introduced '.' as obs-phrase to handle + # existing practice (periods in display names), even though it was not + # allowed in RFC 822. RFC 5322 section 4.1 (which obsoletes RFC 2822) + # continues this requirement. We must recognize obsolete syntax, so + # allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index 95e79b8938b..e23843df448 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -380,7 +380,7 @@ def _fold(self, name, value, sanitize): h = value if h is not None: # The Header class interprets a value of None for maxlinelen as the - # default value of 78, as recommended by RFC 2822. + # default value of 78, as recommended by RFC 5322 section 2.1.1. maxlinelen = 0 if self.max_line_length is not None: maxlinelen = self.max_line_length diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 9d80a5822af..6479b9bab7a 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -32,7 +32,7 @@ NLCRE_bol = re.compile(r'(\r\n|\r|\n)') NLCRE_eol = re.compile(r'(\r\n|\r|\n)\z') NLCRE_crack = re.compile(r'(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character +# RFC 5322 section 3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' @@ -294,7 +294,7 @@ def _parsegen(self): return if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. + # another RFC 5322 message. for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ab5bd0653e4..03524c96559 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -50,7 +50,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. + by RFC 5322 section 2.1.1. The policy keyword specifies a policy object that controls a number of aspects of the generator's operation. If no policy is specified, diff --git a/Lib/email/message.py b/Lib/email/message.py index 4380e0ec50b..641fb2e944d 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -141,7 +141,7 @@ def _decode_uu(encoded): class Message: """Basic message object. - A message object is defined as something that has a bunch of RFC 2822 + A message object is defined as something that has a bunch of RFC 5322 headers and a payload. It may optionally have an envelope header (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a multipart or a message/rfc822), then the payload is a list of Message diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 039f03cba74..c6a51dd8e37 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -2,7 +2,7 @@ # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org -"""A parser of RFC 2822 and MIME email messages.""" +"""A parser of RFC 5322 and MIME email messages.""" __all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', 'FeedParser', 'BytesFeedParser'] @@ -15,13 +15,13 @@ class Parser: def __init__(self, _class=None, *, policy=compat32): - """Parser of RFC 2822 and MIME email messages. + """Parser of RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The string must be formatted as a block of RFC 2822 headers and header + The string must be formatted as a block of RFC 5322 headers and header continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -75,13 +75,13 @@ def parsestr(self, text, headersonly=True): class BytesParser: def __init__(self, *args, **kw): - """Parser of binary RFC 2822 and MIME email messages. + """Parser of binary RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The input must be formatted as a block of RFC 2822 headers and header + The input must be formatted as a block of RFC 5322 headers and header continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/http/client.py b/Lib/http/client.py index 425d9bdad8c..4b9a61cfc11 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -231,7 +231,7 @@ def _read_headers(fp, max_headers): def _parse_header_lines(header_lines, _class=HTTPMessage): """ - Parses only RFC2822 headers from header lines. + Parses only RFC 5322 headers from header lines. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes @@ -244,7 +244,7 @@ def _parse_header_lines(header_lines, _class=HTTPMessage): return email.parser.Parser(_class=_class).parsestr(hstring) def parse_headers(fp, _class=HTTPMessage, *, _max_headers=None): - """Parses only RFC2822 headers from a file pointer.""" + """Parses only RFC 5322 headers from a file pointer.""" headers = _read_headers(fp, _max_headers) return _parse_header_lines(headers, _class) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 808f0fd47e8..72093f7f8b0 100644 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -917,7 +917,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as - described in RFC 2822 (a ValueError is raised if there is more than + described in RFC 5322 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message @@ -931,7 +931,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, policy. """ - # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, @@ -950,7 +950,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: - # Prefer the sender field per RFC 2822:3.6.2. + # Prefer the sender field per RFC 5322 section 3.6.2. from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) diff --git a/Lib/test/test_email/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt index be7d5a2f7b9..0e2bbcaf718 100644 --- a/Lib/test/test_email/data/msg_35.txt +++ b/Lib/test/test_email/data/msg_35.txt @@ -1,4 +1,4 @@ From: aperson@dom.ain To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index b458d3f0efa..4cd587bcd76 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -2373,7 +2373,7 @@ def test_no_separating_blank_line(self): To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here """) # test_defect_handling @@ -2529,49 +2529,49 @@ def test_rfc2047_Q_invalid_digits(self): [(b'andr\xe9=zz', 'iso-8859-1')]) def test_rfc2047_rfc2047_1(self): - # 1st testcase at end of rfc2047 + # 1st testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_2(self): - # 2nd testcase at end of rfc2047 + # 2nd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= b)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b)', None)]) def test_rfc2047_rfc2047_3(self): - # 3rd testcase at end of rfc2047 + # 3rd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_4(self): - # 4th testcase at end of rfc2047 + # 4th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5a(self): - # 5th testcase at end of rfc2047 newline is \r\n + # 5th testcase at end of RFC 2047 newline is \r\n s = '(=?ISO-8859-1?Q?a?=\r\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5b(self): - # 5th testcase at end of rfc2047 newline is \n + # 5th testcase at end of RFC 2047 newline is \n s = '(=?ISO-8859-1?Q?a?=\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_6(self): - # 6th testcase at end of rfc2047 + # 6th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a b', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_7(self): - # 7th testcase at end of rfc2047 + # 7th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b', 'iso-8859-2'), @@ -3273,8 +3273,8 @@ def test_parsedate_y2k(self): """Test for parsing a date with a two-digit year. Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. + four-digit year. RFC 822 allows two-digit years, but RFC 5322 (which + obsoletes RFC 2822, which obsoletes RFC 822) requires four-digit years. """ self.assertEqual(utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), @@ -3325,7 +3325,7 @@ def test_escape_backslashes(self): self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) def test_quotes_unicode_names(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3335,7 +3335,7 @@ def test_quotes_unicode_names(self): latin1_quopri) def test_accepts_any_charset_like_object(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3350,7 +3350,7 @@ def header_encode(self, string): utf8_base64) def test_invalid_charset_like_object_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' # An object without a header_encode method: @@ -3359,7 +3359,7 @@ def test_invalid_charset_like_object_raises_error(self): bad_charset) def test_unicode_address_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. addr = 'pers\u00f6n@dom.in' self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) @@ -3380,7 +3380,7 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): # string containing a quoted backslash, followed by 'example' and two # backslashes, followed by another quoted string containing a space and # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may + # literally. Per RFC 5322 this is not technically correct since a \ may # not appear in an address outside of a quoted string. It is probably # a sensible Postel interpretation, though. eq = self.assertEqual @@ -3392,12 +3392,12 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): ('', '"\\\\"example\\\\" example"@example.com')) def test_parseaddr_preserves_spaces_in_local_part(self): - # issue 9286. A normal RFC5322 local part should not contain any + # issue 9286. A normal RFC 5322 local part should not contain any # folding white space, but legacy local parts can (they are a sequence # of atoms, not dotatoms). On the other hand we strip whitespace from # before the @ and around dots, on the assumption that the whitespace # around the punctuation is a mistake in what would otherwise be - # an RFC5322 local part. Leading whitespace is, usual, stripped as well. + # an RFC 5322 local part. Leading whitespace is, usual, stripped as well. self.assertEqual(('', "merwok wok@xample.com"), utils.parseaddr("merwok wok@xample.com")) self.assertEqual(('', "merwok wok@xample.com"), From bfe54810c408ff066591d1af0411b1d9c10084b1 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 4 Nov 2025 21:19:06 +0000 Subject: [PATCH 016/417] gh-141004: Document `Py_UNICODE_{HIGH, LOW}_SURROGATE` functions (GH-141019) --- Doc/c-api/unicode.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 22b0a6aff6e..ca7c8bb11a5 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -321,12 +321,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings From d81e1ef0f3a7c63f5d246e4cf918700016b72489 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 4 Nov 2025 22:58:53 +0100 Subject: [PATCH 017/417] gh-138189: Document type slots, and other constants, as part of Limited API (GH-138190) Co-authored-by: Peter Bierma --- Doc/c-api/typeobj.rst | 200 ++++++++++++++++++++++++++ Doc/data/stable_abi.dat | 153 ++++++++++++++++++++ Doc/tools/.nitignore | 1 - Doc/tools/extensions/c_annotations.py | 76 +++++++++- Misc/stable_abi.toml | 4 + Tools/build/stable_abi.py | 2 +- 6 files changed, 430 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 59c26a713e4..9d23aea5734 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -676,6 +676,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc + .. corresponding-type-slot:: Py_tp_dealloc + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); @@ -860,6 +862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -877,6 +881,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -909,6 +915,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -974,6 +982,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -1015,6 +1025,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -1028,6 +1040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -1053,6 +1067,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -1077,6 +1093,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1475,6 +1493,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_doc + .. corresponding-type-slot:: Py_tp_doc + An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`~type.__doc__` attribute on the type and instances of the type. @@ -1486,6 +1506,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse + .. corresponding-type-slot:: Py_tp_traverse + An optional pointer to a traversal function for the garbage collector. This is only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: @@ -1582,6 +1604,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear + .. corresponding-type-slot:: Py_tp_clear + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); @@ -1730,6 +1754,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1832,6 +1858,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1847,6 +1875,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1870,6 +1900,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1884,6 +1916,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1899,6 +1933,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1913,6 +1949,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base + .. corresponding-type-slot:: Py_tp_base + An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. @@ -1985,6 +2023,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -2000,6 +2040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -2060,6 +2102,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -2095,6 +2139,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -2118,6 +2164,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -2157,6 +2205,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); @@ -2186,6 +2236,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2214,6 +2266,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases + .. corresponding-type-slot:: Py_tp_bases + Tuple of base types. This field should be set to ``NULL`` and treated as read-only. @@ -2294,6 +2348,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2308,6 +2364,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize + .. corresponding-type-slot:: Py_tp_finalize + An optional pointer to an instance finalization function. This is the C implementation of the :meth:`~object.__del__` special method. Its signature is:: @@ -2466,6 +2524,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2631,42 +2691,148 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: @@ -2683,12 +2849,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2697,6 +2867,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2720,6 +2892,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2727,18 +2901,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2752,6 +2932,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2761,6 +2943,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2768,6 +2952,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2777,6 +2963,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2802,6 +2990,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2851,6 +3041,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2905,6 +3097,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2916,6 +3110,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2928,6 +3124,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2938,6 +3136,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 7ad5f3ecfab..67b498c4268 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,21 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.7,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, type,PyABIInfo,3.15,,full-abi func,PyABIInfo_Check,3.15,, +macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +macro,PyABIInfo_DEFAULT_FLAGS,3.15,, +macro,PyABIInfo_FREETHREADED,3.15,, +macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +macro,PyABIInfo_GIL,3.15,, +macro,PyABIInfo_STABLE,3.15,, macro,PyABIInfo_VAR,3.15,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, @@ -11,6 +25,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -836,6 +870,14 @@ func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, @@ -882,22 +924,133 @@ func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, func,Py_PACK_FULL_VERSION,3.14,, func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_abi,3.15,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 6fee1c192c3..04e8e5580fc 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -8,7 +8,6 @@ Doc/c-api/init_config.rst Doc/c-api/intro.rst Doc/c-api/module.rst Doc/c-api/stable.rst -Doc/c-api/type.rst Doc/c-api/typeobj.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 089614a1f6c..e04a5f144c4 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -154,7 +154,10 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: node.insert(0, annotation) -def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: +def _stable_abi_annotation( + record: StableABIEntry, + is_corresponding_slot: bool = False, +) -> nodes.emphasis: """Create the Stable ABI annotation. These have two forms: @@ -168,9 +171,28 @@ def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: ... all of which can have "since version X.Y" appended. """ stable_added = record.added - message = sphinx_gettext("Part of the") - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + emph_node = nodes.emphasis('', '', classes=["stableabi"]) + if is_corresponding_slot: + # See "Type slot annotations" in add_annotations + ref_node = addnodes.pending_xref( + "slot ID", + refdomain="c", + reftarget="PyType_Slot", + reftype="type", + refexplicit="True", + ) + ref_node += nodes.Text(sphinx_gettext("slot ID")) + + message = sphinx_gettext("The corresponding") + emph_node += nodes.Text(" " + message + " ") + emph_node += ref_node + emph_node += nodes.Text(" ") + emph_node += nodes.literal(record.name, record.name) + message = sphinx_gettext("is part of the") + emph_node += nodes.Text(" " + message + " ") + else: + message = sphinx_gettext("Part of the") + emph_node += nodes.Text(" " + message + " ") ref_node = addnodes.pending_xref( "Stable ABI", refdomain="std", @@ -265,6 +287,51 @@ def run(self) -> list[nodes.Node]: return [node] +class CorrespondingTypeSlot(SphinxDirective): + """Type slot annotations + + Docs for these are with the corresponding field, for example, + "Py_tp_repr" is documented under "PyTypeObject.tp_repr", with + only a stable ABI note mentioning "Py_tp_repr" (and linking to + docs on how this works). + + If there is no corresponding field, these should be documented as normal + macros. + """ + + has_content = False + + required_arguments = 1 + optional_arguments = 0 + + def run(self) -> list[nodes.Node]: + name = self.arguments[0] + state = self.env.domaindata["c_annotations"] + stable_abi_data = state["stable_abi_data"] + + try: + record = stable_abi_data[name] + except LookupError as err: + raise LookupError( + f"{name} is not part of stable ABI. " + + "Document it as `c:macro::` rather than " + + "`corresponding-type-slot::`." + ) from err + + annotation = _stable_abi_annotation(record, is_corresponding_slot=True) + + node = nodes.paragraph() + content = [ + ".. c:namespace:: NULL", + "", + ".. c:macro:: " + name, + " :no-typesetting:", + ] + self.state.nested_parse(StringList(content), 0, node) + node.insert(0, annotation) + return [node] + + def init_annotations(app: Sphinx) -> None: # Using domaindata is a bit hack-ish, # but allows storing state without a global variable or closure. @@ -281,6 +348,7 @@ def setup(app: Sphinx) -> ExtensionMetadata: app.add_config_value("refcount_file", "", "env", types={str}) app.add_config_value("stable_abi_file", "", "env", types={str}) app.add_directive("limited-api-list", LimitedAPIList) + app.add_directive("corresponding-type-slot", CorrespondingTypeSlot) app.connect("builder-inited", init_annotations) app.connect("doctree-read", add_annotations) diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 4a03cc76f5e..ad0f3704599 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2306,6 +2306,10 @@ added = '3.11' [function.PyMemoryView_FromBuffer] added = '3.11' +[const.Py_bf_getbuffer] + added = '3.11' +[const.Py_bf_releasebuffer] + added = '3.11' # Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) # (they were available with 3.11.0) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 1ddd76cdd9b..39115b331ba 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -232,7 +232,7 @@ def sort_key(item): 'data': 'data', 'struct': 'type', 'macro': 'macro', - # 'const': 'const', # all undocumented + 'const': 'macro', 'typedef': 'type', } From d5b00c74b30a159d8d8c03e152c6cf8e8b1431b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 4 Nov 2025 23:29:15 +0100 Subject: [PATCH 018/417] gh-140454: Normalize the JIT stencils filename on Linux to avoid mismatches between the Makefile and the generator (#140823) --- .../next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst | 3 +++ configure | 4 ++-- configure.ac | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst diff --git a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst new file mode 100644 index 00000000000..4bb132ce01e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst @@ -0,0 +1,3 @@ +When building the JIT, match the jit_stencils filename expectations in +Makefile with the generator script. This avoid needless JIT recompilation +during ``make install``. diff --git a/configure b/configure index 60521492755..8463b5b5e4a 100755 --- a/configure +++ b/configure @@ -34327,10 +34327,10 @@ else case e in #( JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" ;; aarch64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-aarch64-unknown-linux-gnu.h" ;; x86_64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-x86_64-unknown-linux-gnu.h" ;; esac ;; esac diff --git a/configure.ac b/configure.ac index 135492d82e0..df94ae25e63 100644 --- a/configure.ac +++ b/configure.ac @@ -8219,10 +8219,10 @@ AS_VAR_IF([enable_experimental_jit], [no], JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" ;; aarch64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-aarch64-unknown-linux-gnu.h" ;; x86_64-*-linux-gnu) - JIT_STENCILS_H="jit_stencils-$host.h" + JIT_STENCILS_H="jit_stencils-x86_64-unknown-linux-gnu.h" ;; esac]) From fa02422918ac3251cdf88a626f90af260bf5224a Mon Sep 17 00:00:00 2001 From: alex <30386655+alexomics@users.noreply.github.com> Date: Wed, 5 Nov 2025 00:05:49 +0000 Subject: [PATCH 019/417] gh-141007: update string module source code link (#141008) In 3.14, the former string.py became `__init__.py` within a new `string` directory that also contains a new submodule file, `templatelib.py`. --- Doc/library/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 6336a0ec47b..58c836c7382 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -4,7 +4,7 @@ .. module:: string :synopsis: Common string operations. -**Source code:** :source:`Lib/string.py` +**Source code:** :source:`Lib/string/__init__.py` -------------- From 1ae900424b3c888d2b2cc97e6ef780717813d658 Mon Sep 17 00:00:00 2001 From: Clifford Gama Date: Wed, 5 Nov 2025 02:29:25 +0200 Subject: [PATCH 020/417] Docs: Fix cached calls count in factorial example (gh-140882) --- Doc/library/functools.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 37d9f87e779..1d9ac328f32 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -42,11 +42,11 @@ The :mod:`functools` module defines the following functions: def factorial(n): return n * factorial(n-1) if n else 1 - >>> factorial(10) # no previously cached result, makes 11 recursive calls + >>> factorial(10) # no previously cached result, makes 11 recursive calls 3628800 - >>> factorial(5) # just looks up cached value result + >>> factorial(5) # no new calls, just returns the cached result 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached + >>> factorial(12) # two new recursive calls, factorial(10) is cached 479001600 The cache is threadsafe so that the wrapped function can be used in From 335d83ec0492779b7fbf2293690f06971cc9d04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=AA=20Nam=20Kh=C3=A1nh?= <55955273+khanhkhanhlele@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:23:25 +0700 Subject: [PATCH 021/417] Fix typo in Apple/__main__.py (#141038) Corrected a typo in a return value docstring. --- Apple/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apple/__main__.py b/Apple/__main__.py index 34744871f68..e76fc351798 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -507,7 +507,7 @@ def lib_non_platform_files(dirname, names): def create_xcframework(platform: str) -> str: """Build an XCframework from the component parts for the platform. - :return: The version number of the Python verion that was packaged. + :return: The version number of the Python version that was packaged. """ package_path = CROSS_BUILD_DIR / platform try: From f2bce51b984f52db14d90f7bbd0b7df00b7c5637 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 5 Nov 2025 11:52:11 +0100 Subject: [PATCH 022/417] gh-140691: urllib.request: Close FTP control socket if data socket can't connect (GH-140835) Co-authored-by: codenamenam --- Lib/_py_warnings.py | 3 + Lib/test/test_urllib2net.py | 59 +++++++++++++++---- Lib/urllib/request.py | 24 +++++--- ...-10-31-15-06-26.gh-issue-140691.JzHGtg.rst | 3 + 4 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 91a9f44b201..67c74fdd2d0 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -646,6 +646,9 @@ def __str__(self): "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + class catch_warnings(object): diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index 0c5f99ec18b..17db686942f 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -1,9 +1,13 @@ +import contextlib import errno +import sysconfig import unittest +from unittest import mock from test import support from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied +from test.support.warnings_helper import check_no_resource_warning import os import socket @@ -143,6 +147,43 @@ def test_ftp(self): ] self._test_urls(urls, self._extra_handlers()) + @support.requires_resource('walltime') + @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le', + 'leaks on PPC64LE (gh-140691)') + def test_ftp_no_leak(self): + # gh-140691: When the data connection (but not control connection) + # cannot be made established, we shouldn't leave an open socket object. + + class MockError(OSError): + pass + + orig_create_connection = socket.create_connection + def patched_create_connection(address, *args, **kwargs): + """Simulate REJECTing connections to ports other than 21""" + host, port = address + if port != 21: + raise MockError() + return orig_create_connection(address, *args, **kwargs) + + url = 'ftp://www.pythontest.net/README' + entry = url, None, urllib.error.URLError + no_cache_handlers = [urllib.request.FTPHandler()] + cache_handlers = self._extra_handlers() + with mock.patch('socket.create_connection', patched_create_connection): + with check_no_resource_warning(self): + # Try without CacheFTPHandler + self._test_urls([entry], handlers=no_cache_handlers, + retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (uncached) + self._test_urls([entry], cache_handlers, retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (cached) + self._test_urls([entry], cache_handlers, retry=False) + # Try without the mock: the handler should not use a closed connection + with check_no_resource_warning(self): + self._test_urls([url], cache_handlers, retry=False) + def test_file(self): TESTFN = os_helper.TESTFN f = open(TESTFN, 'w') @@ -234,18 +275,16 @@ def _test_urls(self, urls, handlers, retry=True): else: req = expected_err = None + if expected_err: + context = self.assertRaises(expected_err) + else: + context = contextlib.nullcontext() + with socket_helper.transient_internet(url): - try: + f = None + with context: f = urlopen(url, req, support.INTERNET_TIMEOUT) - # urllib.error.URLError is a subclass of OSError - except OSError as err: - if expected_err: - msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % - (expected_err, url, req, type(err), err)) - self.assertIsInstance(err, expected_err, msg) - else: - raise - else: + if f is not None: try: with time_out, \ socket_peer_reset, \ diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index af93d4cd75d..566b8087aec 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1535,6 +1535,7 @@ def ftp_open(self, req): dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] + fw = None try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' @@ -1552,8 +1553,12 @@ def ftp_open(self, req): headers += "Content-length: %d\n" % retrlen headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) - except ftplib.all_errors as exp: - raise URLError(f"ftp error: {exp}") from exp + except Exception as exp: + if fw is not None and not fw.keepalive: + fw.close() + if isinstance(exp, ftplib.all_errors): + raise URLError(f"ftp error: {exp}") from exp + raise def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1577,14 +1582,15 @@ def setMaxConns(self, m): def connect_ftp(self, user, passwd, host, port, dirs, timeout): key = user, host, port, '/'.join(dirs), timeout - if key in self.cache: - self.timeout[key] = time.time() + self.delay - else: - self.cache[key] = ftpwrapper(user, passwd, host, port, - dirs, timeout) - self.timeout[key] = time.time() + self.delay + conn = self.cache.get(key) + if conn is None or not conn.keepalive: + if conn is not None: + conn.close() + conn = self.cache[key] = ftpwrapper(user, passwd, host, port, + dirs, timeout) + self.timeout[key] = time.time() + self.delay self.check_cache() - return self.cache[key] + return conn def check_cache(self): # first check for old ones diff --git a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst new file mode 100644 index 00000000000..84b6195c926 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst @@ -0,0 +1,3 @@ +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. From 589a03a8ce60cc65f91930f7d63367b03cfbbb12 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 5 Nov 2025 12:31:42 +0100 Subject: [PATCH 023/417] =?UTF-8?q?gh-140550:=20Initial=20implementation?= =?UTF-8?q?=20of=20PEP=20793=20=E2=80=93=20PyModExport=20(GH-140556)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Victor Stinner Co-authored-by: Kumar Aditya --- Doc/data/stable_abi.dat | 6 + Include/exports.h | 17 +- Include/internal/pycore_importdl.h | 16 +- Include/internal/pycore_moduleobject.h | 36 +- Include/moduleobject.h | 17 +- Include/object.h | 5 + Lib/test/test_capi/test_module.py | 185 +++++++++ Lib/test/test_capi/test_type.py | 18 + Lib/test/test_cext/__init__.py | 1 - Lib/test/test_cext/create_moduledef.c | 29 -- Lib/test/test_cext/extension.c | 54 +-- Lib/test/test_cext/setup.py | 1 - Lib/test/test_import/__init__.py | 92 +++++ Lib/test/test_stable_abi_ctypes.py | 5 + Lib/test/test_sys.py | 5 +- ...-10-26-16-45-28.gh-issue-140556.s__Dae.rst | 2 + Misc/stable_abi.toml | 28 ++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/heaptype.c | 16 + Modules/_testcapi/module.c | 378 ++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 + Modules/_testinternalcapi.c | 29 ++ Modules/_testmultiphase.c | 175 ++++++++ Modules/_testsinglephase.c | 2 + Objects/moduleobject.c | 361 +++++++++++++---- Objects/typeobject.c | 24 +- PC/python3dll.c | 5 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + Python/import.c | 113 ++++-- Python/importdl.c | 100 +++-- 32 files changed, 1494 insertions(+), 236 deletions(-) create mode 100644 Lib/test/test_capi/test_module.py delete mode 100644 Lib/test/test_cext/create_moduledef.c create mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst create mode 100644 Modules/_testcapi/module.c diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 67b498c4268..1359cfa4fbf 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -425,6 +425,7 @@ func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, data,PyLong_Type,3.2,, +macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, func,PyMapping_Check,3.2,, func,PyMapping_GetItemString,3.2,, @@ -471,8 +472,10 @@ func,PyModule_AddObjectRef,3.10,, func,PyModule_AddStringConstant,3.2,, func,PyModule_AddType,3.10,, func,PyModule_Create2,3.2,, +func,PyModule_Exec,3.15,, func,PyModule_ExecDef,3.7,, func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_FromSlotsAndSpec,3.15,, func,PyModule_GetDef,3.2,, func,PyModule_GetDict,3.2,, func,PyModule_GetFilename,3.2,, @@ -480,6 +483,8 @@ func,PyModule_GetFilenameObject,3.2,, func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, +func,PyModule_GetStateSize,3.15,, +func,PyModule_GetToken,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -738,6 +743,7 @@ func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleByToken,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, func,PyType_GetName,3.11,, diff --git a/Include/exports.h b/Include/exports.h index 0c646d5beb6..62feb09ed2b 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -9,6 +9,7 @@ inside the Python core, they are private to the core. If in an extension module, it may be declared with external linkage depending on the platform. + PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array As a number of platforms support/require "__declspec(dllimport/dllexport)", we support a HAVE_DECLSPEC_DLL macro to save duplication. @@ -62,9 +63,9 @@ /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* +# define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ @@ -78,9 +79,9 @@ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ # endif /* Py_BUILD_CORE */ # endif /* HAVE_DECLSPEC_DLL */ @@ -93,13 +94,15 @@ #ifndef PyAPI_DATA # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE #endif -#ifndef PyMODINIT_FUNC +#ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif +#define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* #endif /* Py_EXPORTS_H */ diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 3ba9229cc21..12a32a5f70e 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -28,6 +28,11 @@ typedef enum ext_module_origin { _Py_ext_module_origin_DYNAMIC = 3, } _Py_ext_module_origin; +struct hook_prefixes { + const char *const init_prefix; + const char *const export_prefix; +}; + /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; @@ -40,7 +45,7 @@ struct _Py_ext_module_loader_info { * depending on if it's builtin or not. */ PyObject *path; _Py_ext_module_origin origin; - const char *hook_prefix; + const struct hook_prefixes *hook_prefixes; const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( @@ -62,7 +67,9 @@ extern int _Py_ext_module_loader_info_init_from_spec( PyObject *spec); #endif -/* The result from running an extension module's init function. */ +/* The result from running an extension module's init function. + * Not used for modules defined via PyModExport (slots array). + */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; @@ -89,10 +96,11 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +typedef PyModuleDef_Slot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING -extern PyModInitFunction _PyImport_GetModInitFunc( +extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, - FILE *fp); + FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport); #endif extern int _PyImport_RunModInitFunc( PyModInitFunction p0, diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index b170d7bce70..c34e42e826e 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODULEOBJECT_H #define Py_INTERNAL_MODULEOBJECT_H + +#include + #ifdef __cplusplus extern "C" { #endif @@ -16,32 +19,49 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +typedef int (*_Py_modexecfunc)(PyObject *); + typedef struct { PyObject_HEAD PyObject *md_dict; - PyModuleDef *md_def; void *md_state; PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; + bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED void *md_gil; #endif + Py_ssize_t md_state_size; + traverseproc md_state_traverse; + inquiry md_state_clear; + freefunc md_state_free; + void *md_token; + _Py_modexecfunc md_exec; /* only set if md_token_is_def is true */ } PyModuleObject; -static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_def; +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) + +static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + if (mod->md_token_is_def) { + return (PyModuleDef *)mod->md_token; + } + return NULL; +} + +static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + return mod->md_token; } static inline void* _PyModule_GetState(PyObject* mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_state; + return _PyModule_CAST(mod)->md_state; } static inline PyObject* _PyModule_GetDict(PyObject *mod) { - assert(PyModule_Check(mod)); - PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + PyObject *dict = _PyModule_CAST(mod)->md_dict; // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) assert(dict != NULL); return dict; // borrowed reference diff --git a/Include/moduleobject.h b/Include/moduleobject.h index e3afac0a343..e83bc395aa4 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -83,11 +83,19 @@ struct PyModuleDef_Slot { #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) # define Py_mod_abi 5 +# define Py_mod_name 6 +# define Py_mod_doc 7 +# define Py_mod_state_size 8 +# define Py_mod_methods 9 +# define Py_mod_state_traverse 10 +# define Py_mod_state_clear 11 +# define Py_mod_state_free 12 +# define Py_mod_token 13 #endif #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 5 +#define _Py_mod_LAST_SLOT 13 #endif #endif /* New in 3.5 */ @@ -109,6 +117,13 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *, + PyObject *spec); +PyAPI_FUNC(int) PyModule_Exec(PyObject *mod); +PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *mod, Py_ssize_t *result); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *, void **result); +#endif #ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { diff --git a/Include/object.h b/Include/object.h index 291e4f0a7ed..f17dcba4f47 100644 --- a/Include/object.h +++ b/Include/object.h @@ -839,6 +839,11 @@ PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, + const void *token); +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py new file mode 100644 index 00000000000..7ec23e637d7 --- /dev/null +++ b/Lib/test/test_capi/test_module.py @@ -0,0 +1,185 @@ +# The C functions used by this module are in: +# Modules/_testcapi/module.c + +import unittest +import types +from test.support import import_helper, subTests + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + +class FakeSpec: + name = 'testmod' + +DEF_SLOTS = ( + 'Py_mod_name', 'Py_mod_doc', 'Py_mod_state_size', 'Py_mod_methods', + 'Py_mod_state_traverse', 'Py_mod_state_clear', 'Py_mod_state_free', + 'Py_mod_token', +) + +def def_and_token(mod): + return ( + _testcapi.pymodule_get_def(mod), + _testcapi.pymodule_get_token(mod), + ) + +class TestModFromSlotsAndSpec(unittest.TestCase): + def test_empty(self): + mod = _testcapi.module_from_slots_empty(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, 0) + + def test_null_slots(self): + with self.assertRaises(SystemError): + _testcapi.module_from_slots_null(FakeSpec()) + + def test_none_spec(self): + # The spec currently must contain a name + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_empty(None) + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_name(None) + + def test_name(self): + # Py_mod_name (and PyModuleDef.m_name) are currently ignored when + # spec is given. + # We still test that it's accepted. + mod = _testcapi.module_from_slots_name(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_doc(self): + mod = _testcapi.module_from_slots_doc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, 'the docstring') + + def test_size(self): + mod = _testcapi.module_from_slots_size(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, 123) + + def test_methods(self): + mod = _testcapi.module_from_slots_methods(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_method(456), (mod, 456)) + + def test_gc(self): + mod = _testcapi.module_from_slots_gc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + # Check that the requested hook functions (which module_from_slots_gc + # stores as attributes) match what's in the module (as retrieved by + # _testinternalcapi.module_get_gc_hooks) + _testinternalcapi = import_helper.import_module('_testinternalcapi') + traverse, clear, free = _testinternalcapi.module_get_gc_hooks(mod) + self.assertEqual(traverse, mod.traverse) + self.assertEqual(clear, mod.clear) + self.assertEqual(free, mod.free) + + def test_token(self): + mod = _testcapi.module_from_slots_token(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, _testcapi.module_test_token)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_exec(self): + mod = _testcapi.module_from_slots_exec(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_number, 456) + + def test_create(self): + spec = FakeSpec() + spec._gimme_this = "not a module object" + mod = _testcapi.module_from_slots_create(spec) + self.assertIsInstance(mod, str) + self.assertEqual(mod, "not a module object") + with self.assertRaises(TypeError): + _testcapi.pymodule_get_def(mod), + with self.assertRaises(TypeError): + _testcapi.pymodule_get_token(mod) + + def test_def_slot(self): + """Slots that replace PyModuleDef fields can't be used with PyModuleDef + """ + for name in DEF_SLOTS: + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_def_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("PyModuleDef", str(cm.exception)) + + def test_repeated_def_slot(self): + """Slots that replace PyModuleDef fields can't be repeated""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_repeat_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("more than one", str(cm.exception)) + + def test_null_def_slot(self): + """Slots that replace PyModuleDef fields can't be NULL""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_null_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("NULL", str(cm.exception)) + + def test_def_multiple_exec(self): + """PyModule_Exec runs all exec slots of PyModuleDef-defined module""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + self.assertFalse(hasattr(mod, 'a_number')) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, 789) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, -789) + def_ptr, token = def_and_token(mod) + self.assertEqual(def_ptr, token) + + def test_def_token(self): + """In PyModuleDef-defined modules, the def is the token""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + def_ptr, token = def_and_token(mod) + self.assertEqual(def_ptr, token) + self.assertGreater(def_ptr, 0) + + @subTests('name, expected_size', [ + (__name__, 0), # Python module + ('_testsinglephase', -1), # single-phase init + ('sys', -1), + ]) + def test_get_state_size(self, name, expected_size): + mod = import_helper.import_module(name) + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, expected_size) diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 93874fbee32..e6a8ef9eed6 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -195,6 +195,24 @@ class H2(int): pass with self.assertRaises(TypeError): _testcapi.pytype_getmodulebydef(H2) + def test_get_module_by_token(self): + token = _testcapi.pymodule_get_token(_testcapi) + + heaptype = _testcapi.create_type_with_token('_testcapi.H', 0) + mod = _testcapi.pytype_getmodulebytoken(heaptype, token) + self.assertIs(mod, _testcapi) + + class H1(heaptype): pass + mod = _testcapi.pytype_getmodulebytoken(H1, token) + self.assertIs(mod, _testcapi) + + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebytoken(int, token) + + class H2(int): pass + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebytoken(H2, token) + def test_freeze(self): # test PyType_Freeze() type_freeze = _testcapi.type_freeze diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index fb93c6ccbb6..a52c2241f5d 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -14,7 +14,6 @@ SOURCES = [ os.path.join(os.path.dirname(__file__), 'extension.c'), - os.path.join(os.path.dirname(__file__), 'create_moduledef.c'), ] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') diff --git a/Lib/test/test_cext/create_moduledef.c b/Lib/test/test_cext/create_moduledef.c deleted file mode 100644 index 249c3163552..00000000000 --- a/Lib/test/test_cext/create_moduledef.c +++ /dev/null @@ -1,29 +0,0 @@ -// Workaround for testing _Py_OPAQUE_PYOBJECT. -// See end of 'extension.c' - - -#undef _Py_OPAQUE_PYOBJECT -#undef Py_LIMITED_API -#include "Python.h" - - -// (repeated definition to avoid creating a header) -extern PyObject *testcext_create_moduledef( - const char *name, const char *doc, - PyMethodDef *methods, PyModuleDef_Slot *slots); - -PyObject *testcext_create_moduledef( - const char *name, const char *doc, - PyMethodDef *methods, PyModuleDef_Slot *slots) { - - static struct PyModuleDef _testcext_module = { - PyModuleDef_HEAD_INIT, - }; - if (!_testcext_module.m_name) { - _testcext_module.m_name = name; - _testcext_module.m_doc = doc; - _testcext_module.m_methods = methods; - _testcext_module.m_slots = slots; - } - return PyModuleDef_Init(&_testcext_module); -} diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 73fc67ae59d..0f668c1da32 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -78,7 +78,7 @@ _testcext_exec( return 0; } -#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define _FUNC_NAME(NAME) PyModExport_ ## NAME #define FUNC_NAME(NAME) _FUNC_NAME(NAME) // Converting from function pointer to void* has undefined behavior, but @@ -88,58 +88,40 @@ _testcext_exec( _Py_COMP_DIAG_PUSH #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wcast-qual" #elif defined(__clang__) #pragma clang diagnostic ignored "-Wpedantic" +#pragma clang diagnostic ignored "-Wcast-qual" #endif +PyDoc_STRVAR(_testcext_doc, "C test extension."); + static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_name, STR(MODULE_NAME)}, + {Py_mod_doc, (void*)(char*)_testcext_doc}, {Py_mod_exec, (void*)_testcext_exec}, + {Py_mod_methods, _testcext_methods}, {0, NULL} }; _Py_COMP_DIAG_POP -PyDoc_STRVAR(_testcext_doc, "C test extension."); - -#ifndef _Py_OPAQUE_PYOBJECT - -static struct PyModuleDef _testcext_module = { - PyModuleDef_HEAD_INIT, // m_base - STR(MODULE_NAME), // m_name - _testcext_doc, // m_doc - 0, // m_size - _testcext_methods, // m_methods - _testcext_slots, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free -}; - - -PyMODINIT_FUNC +PyMODEXPORT_FUNC FUNC_NAME(MODULE_NAME)(void) { - return PyModuleDef_Init(&_testcext_module); + return _testcext_slots; } -#else // _Py_OPAQUE_PYOBJECT - -// Opaque PyObject means that PyModuleDef is also opaque and cannot be -// declared statically. See PEP 793. -// So, this part of module creation is split into a separate source file -// which uses non-limited API. - -// (repeated definition to avoid creating a header) -extern PyObject *testcext_create_moduledef( - const char *name, const char *doc, - PyMethodDef *methods, PyModuleDef_Slot *slots); +// Also define the soft-deprecated entrypoint to ensure it isn't called +#define _INITFUNC_NAME(NAME) PyInit_ ## NAME +#define INITFUNC_NAME(NAME) _INITFUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(MODULE_NAME)(void) +INITFUNC_NAME(MODULE_NAME)(void) { - return testcext_create_moduledef( - STR(MODULE_NAME), _testcext_doc, _testcext_methods, _testcext_slots); + PyErr_SetString( + PyExc_AssertionError, + "PyInit_* function called while a PyModExport_* one is available"); + return NULL; } - -#endif // _Py_OPAQUE_PYOBJECT diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 4d71e4751f7..67dfddec751 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -99,7 +99,6 @@ def main(): # Define _Py_OPAQUE_PYOBJECT macro if opaque_pyobject: cflags.append(f'-D_Py_OPAQUE_PYOBJECT') - sources.append('create_moduledef.c') if internal: cflags.append('-DTEST_INTERNAL_C_API=1') diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 072021e5959..e87d8b7e7bb 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2497,6 +2497,21 @@ def test_multi_init_extension_per_interpreter_gil_compat(self): self.check_compatible_here( modname, filename, strict=False, isolated=False) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_testmultiphase_exec_multiple(self): + modname = '_testmultiphase_exec_multiple' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + # All three exec's were called. + self.assertEqual(module.a, 1) + self.assertEqual(module.b, 2) + self.assertEqual(module.c, 3) + # They were called in order. + keys = list(module.__dict__) + self.assertLess(keys.index('a'), keys.index('b')) + self.assertLess(keys.index('b'), keys.index('c')) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_python_compat(self): module = 'threading' @@ -3394,6 +3409,83 @@ def test_basic_multiple_interpreters_reset_each(self): # * module's global state was initialized, not reset +@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") +class ModexportTests(unittest.TestCase): + def test_from_modexport(self): + modname = '_test_from_modexport' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + + def test_from_modexport_null(self): + modname = '_test_from_modexport_null' + filename = _testmultiphase.__file__ + with self.assertRaises(SystemError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + def test_from_modexport_exception(self): + modname = '_test_from_modexport_exception' + filename = _testmultiphase.__file__ + with self.assertRaises(ValueError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + def test_from_modexport_create_nonmodule(self): + modname = '_test_from_modexport_create_nonmodule' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertIsInstance(module, str) + + def test_from_modexport_smoke(self): + # General positive test for sundry features + # (PyModule_FromSlotsAndSpec tests exercise these more carefully) + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertEqual(module.__doc__, "the expected docstring") + self.assertEqual(module.number, 147) + self.assertEqual(module.get_state_int(), 258) + self.assertGreater(module.get_test_token(), 0) + + def test_from_modexport_smoke_token(self): + _testcapi = import_module("_testcapi") + + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + token = module.get_test_token() + self.assertEqual(_testcapi.pymodule_get_token(module), token) + + tp = module.Example + self.assertEqual(_testcapi.pytype_getmodulebytoken(tp, token), module) + class Sub(tp): + pass + self.assertEqual(_testcapi.pytype_getmodulebytoken(Sub, token), module) + + def test_from_modexport_empty_slots(self): + # Module to test that: + # - no slots are mandatory for PyModExport + # - the slots array is used as the default token + modname = '_test_from_modexport_empty_slots' + filename = _testmultiphase.__file__ + module = import_extension_from_file( + modname, filename, put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + self.assertEqual(module.__doc__, None) + + _testcapi = import_module("_testcapi") + smoke_mod = import_extension_from_file( + '_test_from_modexport_smoke', filename, put_in_sys_modules=False) + self.assertEqual(_testcapi.pymodule_get_token(module), + smoke_mod.get_modexport_empty_slots()) + @cpython_only class TestMagicNumber(unittest.TestCase): def test_magic_number_endianness(self): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index cbec7e43a7c..7167646ecc6 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -469,8 +469,10 @@ SYMBOL_NAMES = ( "PyModule_AddStringConstant", "PyModule_AddType", "PyModule_Create2", + "PyModule_Exec", "PyModule_ExecDef", "PyModule_FromDefAndSpec2", + "PyModule_FromSlotsAndSpec", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -478,6 +480,8 @@ SYMBOL_NAMES = ( "PyModule_GetName", "PyModule_GetNameObject", "PyModule_GetState", + "PyModule_GetStateSize", + "PyModule_GetToken", "PyModule_New", "PyModule_NewObject", "PyModule_SetDocString", @@ -733,6 +737,7 @@ SYMBOL_NAMES = ( "PyType_GetFullyQualifiedName", "PyType_GetModule", "PyType_GetModuleByDef", + "PyType_GetModuleByToken", "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1198c6d3511..3ceed019ac4 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1725,9 +1725,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - check(unittest, size('PPPPPP')) + md_gil = 'P' else: - check(unittest, size('PPPPP')) + md_gil = '' + check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) # None check(None, size('')) # NotImplementedType diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst new file mode 100644 index 00000000000..61da60903ee --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst @@ -0,0 +1,2 @@ +:pep:`793`: Add a new entry point for C extension modules, +``PyModExport_``. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index ad0f3704599..7ee6cf1dae5 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2611,3 +2611,31 @@ added = '3.15' [const.PyABIInfo_FREETHREADING_AGNOSTIC] added = '3.15' +[function.PyModule_FromSlotsAndSpec] + added = '3.15' +[function.PyModule_Exec] + added = '3.15' +[function.PyModule_GetToken] + added = '3.15' +[function.PyType_GetModuleByToken] + added = '3.15' +[function.PyModule_GetStateSize] + added = '3.15' +[macro.PyMODEXPORT_FUNC] + added = '3.15' +[const.Py_mod_name] + added = '3.15' +[const.Py_mod_doc] + added = '3.15' +[const.Py_mod_state_size] + added = '3.15' +[const.Py_mod_methods] + added = '3.15' +[const.Py_mod_state_traverse] + added = '3.15' +[const.Py_mod_state_clear] + added = '3.15' +[const.Py_mod_state_free] + added = '3.15' +[const.Py_mod_token] + added = '3.15' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b9ffdcc65d1..2c3013e3d0c 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -175,7 +175,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 69dcf072da1..4fdcc850a33 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -528,6 +528,21 @@ pytype_getmodulebydef(PyObject *self, PyObject *type) return Py_XNewRef(mod); } +static PyObject * +pytype_getmodulebytoken(PyObject *self, PyObject *args) +{ + PyObject *type; + PyObject *py_token; + if (!PyArg_ParseTuple(args, "OO", &type, &py_token)) { + return NULL; + } + void *token = PyLong_AsVoidPtr(py_token); + if ((!token) && PyErr_Occurred()) { + return NULL; + } + return PyType_GetModuleByToken((PyTypeObject *)type, token); +} + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -546,6 +561,7 @@ static PyMethodDef TestMethods[] = { {"get_tp_token", get_tp_token, METH_O}, {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {"pytype_getmodulebydef", pytype_getmodulebydef, METH_O}, + {"pytype_getmodulebytoken", pytype_getmodulebytoken, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c new file mode 100644 index 00000000000..9349445351e --- /dev/null +++ b/Modules/_testcapi/module.c @@ -0,0 +1,378 @@ +#include "parts.h" +#include "util.h" + +// Test PyModule_* API + +/* unittest Cases that use these functions are in: + * Lib/test/test_capi/test_module.py + */ + +static PyObject * +module_from_slots_empty(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null(PyObject *self, PyObject *spec) +{ + return PyModule_FromSlotsAndSpec(NULL, spec); +} + +static PyObject * +module_from_slots_name(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_name, "currently ignored..."}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_doc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_doc, "the docstring"}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_size(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_size, (void*)123}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + return mod; +} + +static PyObject * +a_method(PyObject *self, PyObject *arg) +{ + return PyTuple_Pack(2, self, arg); +} + +static PyMethodDef a_methoddef_array[] = { + {"a_method", a_method, METH_O}, + {0}, +}; + +static PyObject * +module_from_slots_methods(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_methods, a_methoddef_array}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static int noop_traverse(PyObject *self, visitproc visit, void *arg) { + return 0; +} +static int noop_clear(PyObject *self) { return 0; } +static void noop_free(void *self) { } + +static PyObject * +module_from_slots_gc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_traverse, noop_traverse}, + {Py_mod_state_clear, noop_clear}, + {Py_mod_state_free, noop_free}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + if (PyModule_Add(mod, "traverse", PyLong_FromVoidPtr(&noop_traverse)) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Add(mod, "clear", PyLong_FromVoidPtr(&noop_clear)) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Add(mod, "free", PyLong_FromVoidPtr(&noop_free)) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static const char test_token; + +static PyObject * +module_from_slots_token(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_token, (void*)&test_token}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + void *got_token; + if (PyModule_GetToken(mod, &got_token) < 0) { + Py_DECREF(mod); + return NULL; + } + assert(got_token == &test_token); + return mod; +} + +static int +simple_exec(PyObject *module) +{ + return PyModule_AddIntConstant(module, "a_number", 456); +} + +static PyObject * +module_from_slots_exec(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + int res = PyObject_HasAttrStringWithError(mod, "a_number"); + if (res < 0) { + Py_DECREF(mod); + return NULL; + } + assert(res == 0); + if (PyModule_Exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static PyObject * +create_attr_from_spec(PyObject *spec, PyObject *def) +{ + assert(!def); + return PyObject_GetAttrString(spec, "_gimme_this"); +} + +static PyObject * +module_from_slots_create(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_create, create_attr_from_spec}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + + +static int +slot_from_object(PyObject *obj) +{ + PyObject *slot_id_obj = PyObject_GetAttrString(obj, "_test_slot_id"); + if (slot_id_obj == NULL) { + return -1; + } + int slot_id = PyLong_AsInt(slot_id_obj); + if (PyErr_Occurred()) { + return -1; + } + return slot_id; +} + +static PyObject * +module_from_slots_repeat_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {slot_id, "anything else"}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, NULL}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_def_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {0}, + }; + PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "currently ignored", + .m_slots = slots, + }; + // PyModuleDef is normally static; the real requirement is that it + // must outlive its module. + // Here, module creation fails, so it's fine on the stack. + PyObject *result = PyModule_FromDefAndSpec(&def, spec); + assert(result == NULL); + return result; +} + +static int +another_exec(PyObject *module) +{ + /* Make sure simple_exec was called */ + assert(PyObject_HasAttrString(module, "a_number")); + + /* Add or negate a global called 'another_number' */ + PyObject *another_number; + if (PyObject_GetOptionalAttrString(module, "another_number", + &another_number) < 0) { + return -1; + } + if (!another_number) { + return PyModule_AddIntConstant(module, "another_number", 789); + } + PyObject *neg_number = PyNumber_Negative(another_number); + Py_DECREF(another_number); + if (!neg_number) { + return -1; + } + int result = PyObject_SetAttrString(module, "another_number", + neg_number); + Py_DECREF(neg_number); + return result; +} + +static PyObject * +module_from_def_multiple_exec(PyObject *self, PyObject *spec) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {Py_mod_exec, another_exec}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "currently ignored", + .m_slots = slots, + }; + return PyModule_FromDefAndSpec(&def, spec); +} + +static PyObject * +pymodule_exec(PyObject *self, PyObject *module) +{ + if (PyModule_Exec(module) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +pymodule_get_token(PyObject *self, PyObject *module) +{ + void *token; + if (PyModule_GetToken(module, &token) < 0) { + return NULL; + } + return PyLong_FromVoidPtr(token); +} + +static PyObject * +pymodule_get_def(PyObject *self, PyObject *module) +{ + PyModuleDef *def = PyModule_GetDef(module); + if (!def && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromVoidPtr(def); +} + +static PyObject * +pymodule_get_state_size(PyObject *self, PyObject *module) +{ + Py_ssize_t size; + if (PyModule_GetStateSize(module, &size) < 0) { + return NULL; + } + return PyLong_FromSsize_t(size); +} + +static PyMethodDef test_methods[] = { + {"module_from_slots_empty", module_from_slots_empty, METH_O}, + {"module_from_slots_null", module_from_slots_null, METH_O}, + {"module_from_slots_name", module_from_slots_name, METH_O}, + {"module_from_slots_doc", module_from_slots_doc, METH_O}, + {"module_from_slots_size", module_from_slots_size, METH_O}, + {"module_from_slots_methods", module_from_slots_methods, METH_O}, + {"module_from_slots_gc", module_from_slots_gc, METH_O}, + {"module_from_slots_token", module_from_slots_token, METH_O}, + {"module_from_slots_exec", module_from_slots_exec, METH_O}, + {"module_from_slots_create", module_from_slots_create, METH_O}, + {"module_from_slots_repeat_slot", module_from_slots_repeat_slot, METH_O}, + {"module_from_slots_null_slot", module_from_slots_null_slot, METH_O}, + {"module_from_def_multiple_exec", module_from_def_multiple_exec, METH_O}, + {"module_from_def_slot", module_from_def_slot, METH_O}, + {"pymodule_get_token", pymodule_get_token, METH_O}, + {"pymodule_get_def", pymodule_get_def, METH_O}, + {"pymodule_get_state_size", pymodule_get_state_size, METH_O}, + {"pymodule_exec", pymodule_exec, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Module(PyObject *m) +{ +#define ADD_INT_MACRO(C) if (PyModule_AddIntConstant(m, #C, C) < 0) return -1; + ADD_INT_MACRO(Py_mod_create); + ADD_INT_MACRO(Py_mod_exec); + ADD_INT_MACRO(Py_mod_multiple_interpreters); + ADD_INT_MACRO(Py_mod_gil); + ADD_INT_MACRO(Py_mod_name); + ADD_INT_MACRO(Py_mod_doc); + ADD_INT_MACRO(Py_mod_state_size); + ADD_INT_MACRO(Py_mod_methods); + ADD_INT_MACRO(Py_mod_state_traverse); + ADD_INT_MACRO(Py_mod_state_clear); + ADD_INT_MACRO(Py_mod_state_free); + ADD_INT_MACRO(Py_mod_token); +#undef ADD_INT_MACRO + if (PyModule_Add(m, "module_test_token", + PyLong_FromVoidPtr((void*)&test_token)) < 0) + { + return -1; + } + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 32915d04bd3..a7feca5bd96 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -66,5 +66,6 @@ int _PyTestCapi_Init_Import(PyObject *mod); int _PyTestCapi_Init_Frame(PyObject *mod); int _PyTestCapi_Init_Type(PyObject *mod); int _PyTestCapi_Init_Function(PyObject *mod); +int _PyTestCapi_Init_Module(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e29b9ae354b..22cd731d410 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3512,6 +3512,9 @@ _testcapi_exec(PyObject *m) if (_PyTestCapi_Init_Function(m) < 0) { return -1; } + if (_PyTestCapi_Init_Module(m) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c2647d405e2..dede05960d7 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2418,6 +2418,34 @@ set_vectorcall_nop(PyObject *self, PyObject *func) Py_RETURN_NONE; } +static PyObject * +module_get_gc_hooks(PyObject *self, PyObject *arg) +{ + PyModuleObject *mod = (PyModuleObject *)arg; + PyObject *traverse = NULL; + PyObject *clear = NULL; + PyObject *free = NULL; + PyObject *result = NULL; + traverse = PyLong_FromVoidPtr(mod->md_state_traverse); + if (!traverse) { + goto finally; + } + clear = PyLong_FromVoidPtr(mod->md_state_clear); + if (!clear) { + goto finally; + } + free = PyLong_FromVoidPtr(mod->md_state_free); + if (!free) { + goto finally; + } + result = PyTuple_FromArray((PyObject*[]){ traverse, clear, free }, 3); +finally: + Py_XDECREF(traverse); + Py_XDECREF(clear); + Py_XDECREF(free); + return result; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2527,6 +2555,7 @@ static PyMethodDef module_functions[] = { #endif {"simple_pending_call", simple_pending_call, METH_O}, {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, + {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bfec0678e2c..220fa888e49 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -850,6 +850,28 @@ PyInit__testmultiphase_exec_unreported_exception(void) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int execfn_a1(PyObject*m) { return PyModule_AddIntConstant(m, "a", 1); } +static int execfn_b2(PyObject*m) { return PyModule_AddIntConstant(m, "b", 2); } +static int execfn_c3(PyObject*m) { return PyModule_AddIntConstant(m, "c", 3); } + +PyMODINIT_FUNC +PyInit__testmultiphase_exec_multiple(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, execfn_a1}, + {Py_mod_exec, execfn_b2}, + {Py_mod_exec, execfn_c3}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0} + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name="_testmultiphase_exec_multiple", + .m_slots=slots, + }; + return PyModuleDef_Init(&def); +} + static int meth_state_access_exec(PyObject *m) { @@ -993,3 +1015,156 @@ PyInit__test_no_multiple_interpreter_slot(void) { return PyModuleDef_Init(&no_multiple_interpreter_slot_def); } + + +/* PyModExport_* hooks */ + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport"}, + {0}, + }; + return slots; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_null(void) +{ + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_null(void) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_exception(void) +{ + PyErr_SetString(PyExc_ValueError, "failed as requested"); + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_exception(void) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +static PyObject * +modexport_create_string(PyObject *spec, PyObject *def) +{ + assert(def == NULL); + return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_create_nonmodule(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_create_nonmodule"}, + {Py_mod_create, modexport_create_string}, + {0}, + }; + return slots; +} + +static PyModuleDef_Slot modexport_empty_slots[] = { + {0}, +}; + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_empty_slots(void) +{ + return modexport_empty_slots; +} + +static int +modexport_smoke_exec(PyObject *mod) +{ + // "magic" values 147 & 258 are expected in the test + if (PyModule_AddIntConstant(mod, "number", 147) < 0) { + return 0; + } + int *state = PyModule_GetState(mod); + if (!state) { + return -1; + } + *state = 258; + + PyObject *tp = PyType_FromModuleAndSpec(mod, &StateAccessType_spec, NULL); + if (PyModule_Add(mod, "Example", tp) < 0) { + return -1; + } + + return 0; +} + +static PyObject * +modexport_smoke_get_state_int(PyObject *mod, PyObject *arg) +{ + int *state = PyModule_GetState(mod); + if (!state) { + return NULL; + } + return PyLong_FromLong(*state); +} + +static const char modexport_smoke_test_token; + +static PyObject * +modexport_smoke_get_test_token(PyObject *mod, PyObject *arg) +{ + return PyLong_FromVoidPtr((void*)&modexport_smoke_test_token); +} + +static PyObject * +modexport_get_empty_slots(PyObject *mod, PyObject *arg) +{ + /* Get the address of modexport_empty_slots. + * This method would be in the `_test_from_modexport_empty_slots` module, + * if it had a methods slot. + */ + return PyLong_FromVoidPtr(&modexport_empty_slots); +} + +static void +modexport_smoke_free(PyObject *mod) +{ + int *state = PyModule_GetState(mod); + if (!state) { + PyErr_FormatUnraisable("Exception ignored in module %R free", mod); + } + assert(*state == 258); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_smoke(void) +{ + static PyMethodDef methods[] = { + {"get_state_int", modexport_smoke_get_state_int, METH_NOARGS}, + {"get_test_token", modexport_smoke_get_test_token, METH_NOARGS}, + {"get_modexport_empty_slots", modexport_get_empty_slots, METH_NOARGS}, + {0}, + }; + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_smoke"}, + {Py_mod_doc, "the expected docstring"}, + {Py_mod_exec, modexport_smoke_exec}, + {Py_mod_state_size, (void*)sizeof(int)}, + {Py_mod_methods, methods}, + {Py_mod_state_free, modexport_smoke_free}, + {Py_mod_token, (void*)&modexport_smoke_test_token}, + {0}, + }; + return slots; +} diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 2c59085d15b..ee38d61b43a 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -244,6 +244,8 @@ static inline module_state * get_module_state(PyObject *module) { PyModuleDef *def = PyModule_GetDef(module); + assert(def); + if (def->m_size == -1) { return &global_state.module; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 0d45c117168..9dee03bdb5e 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -8,7 +8,7 @@ #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -27,6 +27,27 @@ static PyMemberDef module_members[] = { {0} }; +static void +assert_def_missing_or_redundant(PyModuleObject *m) +{ + /* We copy all relevant info into the module object. + * Modules created using a def keep a reference to that (statically + * allocated) def; the info there should match what we have in the module. + */ +#ifndef NDEBUG + if (m->md_token_is_def) { + PyModuleDef *def = (PyModuleDef *)m->md_token; + assert(def); +#define DO_ASSERT(F) assert (def->m_ ## F == m->md_state_ ## F); + DO_ASSERT(size); + DO_ASSERT(traverse); + DO_ASSERT(clear); + DO_ASSERT(free); +#undef DO_ASSERT + } +#endif // NDEBUG +} + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -44,8 +65,14 @@ _PyModule_IsExtension(PyObject *obj) } PyModuleObject *module = (PyModuleObject*)obj; - PyModuleDef *def = module->md_def; - return (def != NULL && def->m_methods != NULL); + if (module->md_exec) { + return 1; + } + if (module->md_token_is_def) { + PyModuleDef *def = (PyModuleDef *)module->md_token; + return (module->md_token_is_def && def->m_methods != NULL); + } + return 0; } @@ -146,10 +173,19 @@ new_module_notrack(PyTypeObject *mt) m = (PyModuleObject *)_PyType_AllocNoTrack(mt, 0); if (m == NULL) return NULL; - m->md_def = NULL; m->md_state = NULL; m->md_weaklist = NULL; m->md_name = NULL; + m->md_token_is_def = false; +#ifdef Py_GIL_DISABLED + m->md_gil = Py_MOD_GIL_USED; +#endif + m->md_state_size = 0; + m->md_state_traverse = NULL; + m->md_state_clear = NULL; + m->md_state_free = NULL; + m->md_exec = NULL; + m->md_token = NULL; m->md_dict = PyDict_New(); if (m->md_dict == NULL) { Py_DECREF(m); @@ -264,6 +300,17 @@ PyModule_Create2(PyModuleDef* module, int module_api_version) return _PyModule_CreateInitialized(module, module_api_version); } +static void +module_copy_members_from_deflike( + PyModuleObject *md, + PyModuleDef *def_like /* not necessarily a valid Python object */) +{ + md->md_state_size = def_like->m_size; + md->md_state_traverse = def_like->m_traverse; + md->md_state_clear = def_like->m_clear; + md->md_state_free = def_like->m_free; +} + PyObject * _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) { @@ -310,15 +357,21 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } } - m->md_def = module; + m->md_token = module; + m->md_token_is_def = true; + module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED m->md_gil = Py_MOD_GIL_USED; #endif return (PyObject*)m; } -PyObject * -PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +static PyObject * +module_from_def_and_spec( + PyModuleDef* def_like, /* not necessarily a valid Python object */ + PyObject *spec, + int module_api_version, + PyModuleDef* original_def /* NULL if not defined by a def */) { PyModuleDef_Slot* cur_slot; PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; @@ -331,10 +384,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio int has_execution_slots = 0; const char *name; int ret; + void *token = NULL; + _Py_modexecfunc m_exec = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); - PyModuleDef_Init(def); - nameobj = PyObject_GetAttrString(spec, "name"); if (nameobj == NULL) { return NULL; @@ -348,7 +401,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - if (def->m_size < 0) { + if (def_like->m_size < 0) { PyErr_Format( PyExc_SystemError, "module %s: m_size may not be negative for multi-phase initialization", @@ -356,7 +409,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + // Macro to copy a non-NULL, non-repeatable slot that's unusable with + // PyModuleDef. The destination must be initially NULL. +#define COPY_COMMON_SLOT(SLOT, TYPE, DEST) \ + do { \ + if (!(TYPE)(cur_slot->value)) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " must not be NULL", \ + name); \ + goto error; \ + } \ + if (original_def) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " used with PyModuleDef", \ + name); \ + goto error; \ + } \ + if (DEST) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s has more than one " #SLOT " slot", \ + name); \ + goto error; \ + } \ + DEST = (TYPE)(cur_slot->value); \ + } while (0); \ + ///////////////////////////////////////////////////////////////// switch (cur_slot->slot) { case Py_mod_create: if (create) { @@ -370,6 +451,9 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio break; case Py_mod_exec: has_execution_slots = 1; + if (!original_def) { + COPY_COMMON_SLOT(Py_mod_exec, _Py_modexecfunc, m_exec); + } break; case Py_mod_multiple_interpreters: if (has_multiple_interpreters_slot) { @@ -398,6 +482,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } break; + case Py_mod_name: + COPY_COMMON_SLOT(Py_mod_name, char*, def_like->m_name); + break; + case Py_mod_doc: + COPY_COMMON_SLOT(Py_mod_doc, char*, def_like->m_doc); + break; + case Py_mod_state_size: + COPY_COMMON_SLOT(Py_mod_state_size, Py_ssize_t, + def_like->m_size); + break; + case Py_mod_methods: + COPY_COMMON_SLOT(Py_mod_methods, PyMethodDef*, + def_like->m_methods); + break; + case Py_mod_state_traverse: + COPY_COMMON_SLOT(Py_mod_state_traverse, traverseproc, + def_like->m_traverse); + break; + case Py_mod_state_clear: + COPY_COMMON_SLOT(Py_mod_state_clear, inquiry, + def_like->m_clear); + break; + case Py_mod_state_free: + COPY_COMMON_SLOT(Py_mod_state_free, freefunc, + def_like->m_free); + break; + case Py_mod_token: + COPY_COMMON_SLOT(Py_mod_token, void*, token); + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -406,6 +519,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name, cur_slot->slot); goto error; } +#undef COPY_COMMON_SLOT } /* By default, multi-phase init modules are expected @@ -429,7 +543,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (create) { - m = create(spec, def); + m = create(spec, original_def); if (m == NULL) { if (!PyErr_Occurred()) { PyErr_Format( @@ -455,15 +569,27 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (PyModule_Check(m)) { - ((PyModuleObject*)m)->md_state = NULL; - ((PyModuleObject*)m)->md_def = def; + PyModuleObject *mod = (PyModuleObject*)m; + mod->md_state = NULL; + module_copy_members_from_deflike(mod, def_like); + if (original_def) { + assert (!token); + mod->md_token = original_def; + mod->md_token_is_def = 1; + } + else { + mod->md_token = token; + } #ifdef Py_GIL_DISABLED - ((PyModuleObject*)m)->md_gil = gil_slot; + mod->md_gil = gil_slot; #else (void)gil_slot; #endif + mod->md_exec = m_exec; } else { - if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { + if (def_like->m_size > 0 || def_like->m_traverse || def_like->m_clear + || def_like->m_free) + { PyErr_Format( PyExc_SystemError, "module %s is not a module object, but requests module state", @@ -478,17 +604,25 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name); goto error; } + if (token) { + PyErr_Format( + PyExc_SystemError, + "module %s specifies a token, but did not create " + "a ModuleType instance", + name); + goto error; + } } - if (def->m_methods != NULL) { - ret = _add_methods_to_object(m, nameobj, def->m_methods); + if (def_like->m_methods != NULL) { + ret = _add_methods_to_object(m, nameobj, def_like->m_methods); if (ret != 0) { goto error; } } - if (def->m_doc != NULL) { - ret = PyModule_SetDocString(m, def->m_doc); + if (def_like->m_doc != NULL) { + ret = PyModule_SetDocString(m, def_like->m_doc); if (ret != 0) { goto error; } @@ -503,6 +637,29 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +PyObject * +PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +{ + PyModuleDef_Init(def); + return module_from_def_and_spec(def, spec, module_api_version, def); +} + +PyObject * +PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +{ + if (!slots) { + PyErr_SetString( + PyExc_SystemError, + "PyModule_FromSlotsAndSpec called with NULL slots"); + return NULL; + } + // Fill in enough of a PyModuleDef to pass to common machinery + PyModuleDef def_like = {.m_slots = (PyModuleDef_Slot *)slots}; + + return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION, + NULL); +} + #ifdef Py_GIL_DISABLED int PyUnstable_Module_SetGIL(PyObject *module, void *gil) @@ -516,71 +673,94 @@ PyUnstable_Module_SetGIL(PyObject *module, void *gil) } #endif -int -PyModule_ExecDef(PyObject *module, PyModuleDef *def) +static int +run_exec_func(PyObject *module, int (*exec)(PyObject *)) { - PyModuleDef_Slot *cur_slot; - const char *name; - int ret; - - name = PyModule_GetName(module); - if (name == NULL) { + int ret = exec(module); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "execution of %R failed without setting an exception", + module); + } return -1; } + if (PyErr_Occurred()) { + _PyErr_FormatFromCause( + PyExc_SystemError, + "execution of module %R raised unreported exception", + module); + return -1; + } + return 0; +} - if (def->m_size >= 0) { - PyModuleObject *md = (PyModuleObject*)module; +static int +alloc_state(PyObject *module) +{ + if (!PyModule_Check(module)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", module); + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + + if (md->md_state_size >= 0) { if (md->md_state == NULL) { /* Always set a state pointer; this serves as a marker to skip * multiple initialization (importlib.reload() is no-op) */ - md->md_state = PyMem_Malloc(def->m_size); + md->md_state = PyMem_Malloc(md->md_state_size); if (!md->md_state) { PyErr_NoMemory(); return -1; } - memset(md->md_state, 0, def->m_size); + memset(md->md_state, 0, md->md_state_size); } } + return 0; +} + +int +PyModule_Exec(PyObject *module) +{ + if (alloc_state(module) < 0) { + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + if (md->md_exec) { + assert(!md->md_token_is_def); + return run_exec_func(module, md->md_exec); + } + + PyModuleDef *def = _PyModule_GetDefOrNull(module); + if (def) { + return PyModule_ExecDef(module, def); + } + return 0; +} + +int +PyModule_ExecDef(PyObject *module, PyModuleDef *def) +{ + PyModuleDef_Slot *cur_slot; + + if (alloc_state(module) < 0) { + return -1; + } + + assert(PyModule_Check(module)); if (def->m_slots == NULL) { return 0; } for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - switch (cur_slot->slot) { - case Py_mod_create: - /* handled in PyModule_FromDefAndSpec2 */ - break; - case Py_mod_exec: - ret = ((int (*)(PyObject *))cur_slot->value)(module); - if (ret != 0) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "execution of module %s failed without setting an exception", - name); - } - return -1; - } - if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "execution of module %s raised unreported exception", - name); - return -1; - } - break; - case Py_mod_multiple_interpreters: - case Py_mod_gil: - case Py_mod_abi: - /* handled in PyModule_FromDefAndSpec2 */ - break; - default: - PyErr_Format( - PyExc_SystemError, - "module %s initialized with unknown slot %i", - name, cur_slot->slot); + if (cur_slot->slot == Py_mod_exec) { + int (*func)(PyObject *) = cur_slot->value; + if (run_exec_func(module, func) < 0) { return -1; + } + continue; } } return 0; @@ -624,6 +804,31 @@ PyModule_GetDict(PyObject *m) return _PyModule_GetDict(m); // borrowed reference } +int +PyModule_GetStateSize(PyObject *m, Py_ssize_t *size_p) +{ + *size_p = -1; + if (!PyModule_Check(m)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", m); + return -1; + } + PyModuleObject *mod = (PyModuleObject *)m; + *size_p = mod->md_state_size; + return 0; +} + +int +PyModule_GetToken(PyObject *m, void **token_p) +{ + *token_p = NULL; + if (!PyModule_Check(m)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", m); + return -1; + } + *token_p = _PyModule_GetToken(m); + return 0; +} + PyObject* PyModule_GetNameObject(PyObject *mod) { @@ -764,7 +969,7 @@ PyModule_GetDef(PyObject* m) PyErr_BadArgument(); return NULL; } - return _PyModule_GetDef(m); + return _PyModule_GetDefOrNull(m); } void* @@ -888,17 +1093,18 @@ module_dealloc(PyObject *self) } FT_CLEAR_WEAKREFS(self, m->md_weaklist); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_free - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_free && (m->md_state_size <= 0 || m->md_state != NULL)) { - m->md_def->m_free(m); + m->md_state_free(m); } Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); - if (m->md_state != NULL) + if (m->md_state != NULL) { PyMem_Free(m->md_state); + } Py_TYPE(m)->tp_free((PyObject *)m); } @@ -1206,11 +1412,11 @@ module_traverse(PyObject *self, visitproc visit, void *arg) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_traverse - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_traverse && (m->md_state_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_traverse((PyObject*)m, visit, arg); + int res = m->md_state_traverse((PyObject*)m, visit, arg); if (res) return res; } @@ -1224,18 +1430,19 @@ module_clear(PyObject *self) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_clear - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_clear && (m->md_state_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_clear((PyObject*)m); + int res = m->md_state_clear((PyObject*)m); if (PyErr_Occurred()) { PyErr_FormatUnraisable("Exception ignored in m_clear of module%s%V", m->md_name ? " " : "", m->md_name, ""); } - if (res) + if (res) { return res; + } } Py_CLEAR(m->md_dict); return 0; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d5695015807..326f4add896 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5764,11 +5764,11 @@ PyType_GetModuleState(PyTypeObject *type) } -/* Get the module of the first superclass where the module has the - * given PyModuleDef. +/* Return borrowed ref to the module of the first superclass where the module + * has the given token. */ -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +static PyObject * +borrow_module_by_token(PyTypeObject *type, const void *token) { assert(PyType_Check(type)); @@ -5780,7 +5780,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) else { PyHeapTypeObject *ht = (PyHeapTypeObject*)type; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetToken(module) == token) { return module; } } @@ -5808,7 +5808,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetToken(module) == token) { res = module; break; } @@ -5826,6 +5826,18 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return NULL; } +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + return borrow_module_by_token(type, def); +} + +PyObject * +PyType_GetModuleByToken(PyTypeObject *type, const void *token) +{ + return Py_XNewRef(borrow_module_by_token(type, token)); +} + static PyTypeObject * get_base_by_token_recursive(PyObject *bases, void *token) diff --git a/PC/python3dll.c b/PC/python3dll.c index 05c86e6d592..99e0f05fe03 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -416,8 +416,10 @@ EXPORT_FUNC(PyModule_AddObjectRef) EXPORT_FUNC(PyModule_AddStringConstant) EXPORT_FUNC(PyModule_AddType) EXPORT_FUNC(PyModule_Create2) +EXPORT_FUNC(PyModule_Exec) EXPORT_FUNC(PyModule_ExecDef) EXPORT_FUNC(PyModule_FromDefAndSpec2) +EXPORT_FUNC(PyModule_FromSlotsAndSpec) EXPORT_FUNC(PyModule_GetDef) EXPORT_FUNC(PyModule_GetDict) EXPORT_FUNC(PyModule_GetFilename) @@ -425,6 +427,8 @@ EXPORT_FUNC(PyModule_GetFilenameObject) EXPORT_FUNC(PyModule_GetName) EXPORT_FUNC(PyModule_GetNameObject) EXPORT_FUNC(PyModule_GetState) +EXPORT_FUNC(PyModule_GetStateSize) +EXPORT_FUNC(PyModule_GetToken) EXPORT_FUNC(PyModule_New) EXPORT_FUNC(PyModule_NewObject) EXPORT_FUNC(PyModule_SetDocString) @@ -668,6 +672,7 @@ EXPORT_FUNC(PyType_GetFlags) EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) EXPORT_FUNC(PyType_GetModuleByDef) +EXPORT_FUNC(PyType_GetModuleByToken) EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index a355a5fc257..68707a54ff6 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -126,6 +126,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 05128d3ac36..b0e75ce433a 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -111,6 +111,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/import.c b/Python/import.c index d4b574a8828..6cf4a061ca6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -672,8 +672,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (6). first time (not found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() - C. _PyImport_GetModInitFunc(): load + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load D. _imp_create_dynamic_impl() -> import_run_extension() E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call @@ -743,16 +743,19 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) A. noop - ...for multi-phase init modules: + ...for multi-phase init modules from PyModInit_* (PyModuleDef): (6). every time: A. _imp_create_dynamic_impl() -> import_find_extension() (not found) - B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() - C. _PyImport_GetModInitFunc(): load + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load D. _imp_create_dynamic_impl() -> import_run_extension() E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call G. import_run_extension() -> PyModule_FromDefAndSpec() + + PyModule_FromDefAndSpec(): + H. PyModule_FromDefAndSpec(): gather/check moduledef slots I. if there's a Py_mod_create slot: 1. PyModule_FromDefAndSpec(): call its function @@ -765,10 +768,29 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (10). every time: A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() B. if mod->md_state == NULL (including if m_size == 0): - 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() - 2. PyModule_ExecDef(): allocate mod->md_state + 1. exec_builtin_or_dynamic() -> PyModule_Exec() + 2. PyModule_Exec(): allocate mod->md_state 3. if there's a Py_mod_exec slot: - 1. PyModule_ExecDef(): call its function + 1. PyModule_Exec(): call its function + + + ...for multi-phase init modules from PyModExport_* (slots array): + + (6). every time: + + A. _imp_create_dynamic_impl() -> import_find_extension() (not found) + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load + D. _imp_create_dynamic_impl() -> import_run_modexport() + E. import_run_modexport(): call + F. import_run_modexport() -> PyModule_FromSlotsAndSpec() + G. PyModule_FromSlotsAndSpec(): create temporary PyModuleDef-like + H. PyModule_FromSlotsAndSpec() -> PyModule_FromDefAndSpec() + + (PyModule_FromDefAndSpec behaves as for PyModInit_*, above) + + (10). every time: as for PyModInit_*, above + */ @@ -825,25 +847,19 @@ _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ static int exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; void *state; if (!PyModule_Check(mod)) { return 0; } - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - state = PyModule_GetState(mod); if (state) { /* Already initialized; skip reload */ return 0; } - return PyModule_ExecDef(mod, def); + return PyModule_Exec(mod); } @@ -1787,7 +1803,7 @@ finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(cached->def == _PyModule_GetDef(mod)); + assert(cached->def == _PyModule_GetDefOrNull(mod)); Py_ssize_t index = _get_cached_module_index(cached); if (_modules_by_index_set(tstate->interp, index, mod) < 0) { @@ -1865,8 +1881,8 @@ reload_singlephase_extension(PyThreadState *tstate, * due to violating interpreter isolation. * See the note in set_cached_m_dict(). * Until that is solved, we leave md_def set to NULL. */ - assert(_PyModule_GetDef(mod) == NULL - || _PyModule_GetDef(mod) == def); + assert(_PyModule_GetDefOrNull(mod) == NULL + || _PyModule_GetDefOrNull(mod) == def); } else { assert(cached->m_dict == NULL); @@ -1953,6 +1969,43 @@ import_find_extension(PyThreadState *tstate, return mod; } +static PyObject * +import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, + struct _Py_ext_module_loader_info *info, + PyObject *spec) +{ + /* This is like import_run_extension, but avoids interpreter switching + * and code for for single-phase modules. + */ + PyModuleDef_Slot *slots = ex0(); + if (!slots) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s failed without setting an exception", + info->name); + } + return NULL; + } + if (PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s raised unreported exception", + info->name); + } + PyObject *result = PyModule_FromSlotsAndSpec(slots, spec); + if (!result) { + return NULL; + } + if (PyModule_Check(result)) { + PyModuleObject *mod = (PyModuleObject *)result; + if (mod && !mod->md_token) { + mod->md_token = slots; + } + } + return result; +} + static PyObject * import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_info *info, @@ -2125,7 +2178,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, assert_multiphase_def(def); assert(mod == NULL); /* Note that we cheat a little by not repeating the calls - * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ + * to _PyImport_GetModuleExportHooks() and _PyImport_RunModInitFunc(). */ mod = PyModule_FromDefAndSpec(def, spec); if (mod == NULL) { goto error; @@ -2239,8 +2292,9 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, return -1; } - PyModuleDef *def = PyModule_GetDef(mod); + PyModuleDef *def = _PyModule_GetDefOrNull(mod); if (def == NULL) { + assert(!PyErr_Occurred()); PyErr_BadInternalCall(); goto finally; } @@ -2322,8 +2376,8 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4653,8 +4707,8 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4679,7 +4733,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } /* We would move this (and the fclose() below) into - * _PyImport_GetModInitFunc(), but it isn't clear if the intervening + * _PyImport_GetModuleExportHooks(), but it isn't clear if the intervening * code relies on fp still being open. */ FILE *fp; if (file != NULL) { @@ -4692,7 +4746,13 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) fp = NULL; } - PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + PyModInitFunction p0 = NULL; + PyModExportFunction ex0 = NULL; + _PyImport_GetModuleExportHooks(&info, fp, &p0, &ex0); + if (ex0) { + mod = import_run_modexport(tstate, ex0, &info, spec); + goto cleanup; + } if (p0 == NULL) { goto finally; } @@ -4714,6 +4774,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } #endif +cleanup: // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? if (fp) { fclose(fp); diff --git a/Python/importdl.c b/Python/importdl.c index 802843fe7b9..23a55c39677 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -5,7 +5,7 @@ #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_import.h" // _PyImport_SwapPackageContext() #include "pycore_importdl.h" -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_runtime.h" // _Py_ID() @@ -35,8 +35,10 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, /* module info to use when loading */ /***********************************/ -static const char * const ascii_only_prefix = "PyInit"; -static const char * const nonascii_prefix = "PyInitU"; +static const struct hook_prefixes ascii_only_prefixes = { + "PyInit", "PyModExport"}; +static const struct hook_prefixes nonascii_prefixes = { + "PyInitU", "PyModExportU"}; /* Get the variable part of a module's export symbol name. * Returns a bytes instance. For non-ASCII-named modules, the name is @@ -45,7 +47,7 @@ static const char * const nonascii_prefix = "PyInitU"; * nonascii_prefix, as appropriate. */ static PyObject * -get_encoded_name(PyObject *name, const char **hook_prefix) { +get_encoded_name(PyObject *name, const struct hook_prefixes **hook_prefixes) { PyObject *tmp; PyObject *encoded = NULL; PyObject *modname = NULL; @@ -72,7 +74,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { /* Encode to ASCII or Punycode, as needed */ encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); if (encoded != NULL) { - *hook_prefix = ascii_only_prefix; + *hook_prefixes = &ascii_only_prefixes; } else { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { PyErr_Clear(); @@ -80,7 +82,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { if (encoded == NULL) { goto error; } - *hook_prefix = nonascii_prefix; + *hook_prefixes = &nonascii_prefixes; } else { goto error; } @@ -130,7 +132,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, assert(PyUnicode_GetLength(name) > 0); info.name = Py_NewRef(name); - info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + info.name_encoded = get_encoded_name(info.name, &info.hook_prefixes); if (info.name_encoded == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; @@ -189,7 +191,7 @@ _Py_ext_module_loader_info_init_for_builtin( /* We won't need filename. */ .path=name, .origin=_Py_ext_module_origin_BUILTIN, - .hook_prefix=ascii_only_prefix, + .hook_prefixes=&ascii_only_prefixes, .newcontext=NULL, }; return 0; @@ -377,39 +379,63 @@ _Py_ext_module_loader_result_apply_error( /********************************************/ #ifdef HAVE_DYNAMIC_LOADING -PyModInitFunction -_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, - FILE *fp) +static dl_funcptr +findfuncptr(const char *prefix, const char *name_buf, + struct _Py_ext_module_loader_info *info, + FILE *fp) { +#ifdef MS_WINDOWS + return _PyImport_FindSharedFuncptrWindows( + prefix, name_buf, info->filename, fp); +#else + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); + return _PyImport_FindSharedFuncptr( + prefix, name_buf, path_buf, fp); +#endif +} + +int +_PyImport_GetModuleExportHooks( + struct _Py_ext_module_loader_info *info, + FILE *fp, + PyModInitFunction *modinit, + PyModExportFunction *modexport) +{ + *modinit = NULL; + *modexport = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); dl_funcptr exportfunc; -#ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows( - info->hook_prefix, name_buf, info->filename, fp); -#else - { - const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); - exportfunc = _PyImport_FindSharedFuncptr( - info->hook_prefix, name_buf, path_buf, fp); - } -#endif - if (exportfunc == NULL) { - if (!PyErr_Occurred()) { - PyObject *msg; - msg = PyUnicode_FromFormat( - "dynamic module does not define " - "module export function (%s_%s)", - info->hook_prefix, name_buf); - if (msg != NULL) { - PyErr_SetImportError(msg, info->name, info->filename); - Py_DECREF(msg); - } + exportfunc = findfuncptr( + info->hook_prefixes->export_prefix, + name_buf, info, fp); + if (exportfunc) { + *modexport = (PyModExportFunction)exportfunc; + return 2; + } + + exportfunc = findfuncptr( + info->hook_prefixes->init_prefix, + name_buf, info, fp); + if (exportfunc) { + *modinit = (PyModInitFunction)exportfunc; + return 1; + } + + if (!PyErr_Occurred()) { + PyObject *msg; + msg = PyUnicode_FromFormat( + "dynamic module does not define " + "module export function (%s_%s or %s_%s)", + info->hook_prefixes->export_prefix, name_buf, + info->hook_prefixes->init_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info->name, info->filename); + Py_DECREF(msg); } - return NULL; } - - return (PyModInitFunction)exportfunc; + return -1; } #endif /* HAVE_DYNAMIC_LOADING */ @@ -477,7 +503,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, res.def = (PyModuleDef *)m; /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - else if (info->hook_prefix == nonascii_prefix) { + else if (info->hook_prefixes == &nonascii_prefixes) { /* Non-ASCII is only supported for multi-phase init. */ res.kind = _Py_ext_module_kind_MULTIPHASE; /* Don't allow legacy init for non-ASCII module names. */ @@ -496,7 +522,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, goto error; } - res.def = _PyModule_GetDef(m); + res.def = _PyModule_GetDefOrNull(m); if (res.def == NULL) { PyErr_Clear(); _Py_ext_module_loader_result_set_error( From 35528fccdcaa0890e959eb7884332d1a426819ac Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 09:13:57 -0500 Subject: [PATCH 024/417] gh-141004: Document missing iterator types in the C API (GH-141010) Add documentation for each of the following: - PyByteArrayIter_Type - PyBytesIter_Type - PyListIter_Type - PyListRevIter_Type - PySetIter_Type - PyTupleIter_Type - PyRangeIter_Type - PyLongRangeIter_Type - PyDictIterKey_Type - PyDictRevIterKey_Type - PyDictIterValue_Type - PyDictRevIterValue_Type - PyDictIterItem_Type - PyDictRevIterItem_Type --------- Co-authored-by: Petr Viktorin --- Doc/c-api/iterator.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c9979..4b94970036f 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,32 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. From 3f6aca1be49f96c5c5f52040b8e78c73c79c0a86 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 11:45:13 -0500 Subject: [PATCH 025/417] gh-141004: Document `PyMemoryView_Type` (GH-141034) --- Doc/c-api/memoryview.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805..e4ac8b57673 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. From 579b2f8910d6c4b07094d86b01d5421a55b09533 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 17:57:06 +0100 Subject: [PATCH 026/417] gh-140550: Run make regen-limited-abi (#141056) --- Doc/data/stable_abi.dat | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 1359cfa4fbf..5cbf3771950 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -979,6 +979,14 @@ macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, macro,Py_mod_abi,3.15,, +macro,Py_mod_doc,3.15,, +macro,Py_mod_methods,3.15,, +macro,Py_mod_name,3.15,, +macro,Py_mod_state_clear,3.15,, +macro,Py_mod_state_free,3.15,, +macro,Py_mod_state_size,3.15,, +macro,Py_mod_state_traverse,3.15,, +macro,Py_mod_token,3.15,, macro,Py_mp_ass_subscript,3.2,, macro,Py_mp_length,3.2,, macro,Py_mp_subscript,3.2,, From 30ab627aab050840d17ac14c9d3730d065bda6af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 18:31:35 +0100 Subject: [PATCH 027/417] gh-83714: Fix a compiler warning in stat_nanosecond_timestamp() (#141043) Disable the fast path on systems with 32-bit long. --- Modules/posixmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 50464b01efb..ecda75ec6ab 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2634,11 +2634,14 @@ _posix_free(void *module) static PyObject * stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) { +#if SIZEOF_LONG >= 8 /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); } - else { + else +#endif + { PyObject *ns_total = NULL; PyObject *s_in_ns = NULL; PyObject *s = _PyLong_FromTime_t(sec); From 8d55faf2d68bbb6486a3e4509e8912d211748756 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 18:37:06 +0100 Subject: [PATCH 028/417] Remove internal _PyTime_AsLong() function (#141053) * Replace _PyTime_AsLong() with PyLong_FromInt64() * Replace _PyTime_FromLong() with PyLong_AsInt64(). --- Include/internal/pycore_time.h | 9 -------- Modules/_lsprof.c | 4 ++-- Modules/_testinternalcapi/pytime.c | 20 ++++++++--------- Modules/timemodule.c | 14 ++++++------ Python/pytime.c | 35 +----------------------------- 5 files changed, 20 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 23312471c65..b671225ca6e 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -147,11 +147,6 @@ extern int _PyTime_FromSecondsDouble( // Clamp to [PyTime_MIN; PyTime_MAX] on overflow. extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from a Python int object (number of nanoseconds). -// Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, - PyObject *obj); - // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. @@ -182,10 +177,6 @@ extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert a timestamp (number of nanoseconds) as a Python int object. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); - #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index c20dbc3f4f4..025a3fac46e 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -6,7 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_FromLong() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -111,7 +111,7 @@ static PyTime_t CallExternalTimer(ProfilerObject *pObj) if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromLong(&result, o); + err = PyLong_AsInt64(o, &result); } else { /* interpret the result as a double measured in seconds. diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index 2b0a205d158..7fb100c41a1 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -17,7 +17,7 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsLong(ts); + return PyLong_FromInt64(ts); } static int @@ -49,7 +49,7 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsLong(ts); + return PyLong_FromInt64(ts); } static PyObject * @@ -64,7 +64,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timeval tv; @@ -91,7 +91,7 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timeval tv; @@ -113,7 +113,7 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timespec ts; @@ -131,7 +131,7 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timespec ts; @@ -149,14 +149,14 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } PyTime_t ms = _PyTime_AsMilliseconds(t, round); - return _PyTime_AsLong(ms); + return PyLong_FromInt64(ms); } static PyObject * @@ -168,14 +168,14 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } PyTime_t us = _PyTime_AsMicroseconds(t, round); - return _PyTime_AsLong(us); + return PyLong_FromInt64(us); } static PyObject * diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3271d87ddc2..3946d18479e 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -128,7 +128,7 @@ time_time_ns(PyObject *self, PyObject *unused) if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(time_ns_doc, @@ -261,7 +261,7 @@ time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } #endif /* HAVE_CLOCK_GETTIME */ @@ -310,7 +310,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -1216,7 +1216,7 @@ time_monotonic_ns(PyObject *self, PyObject *unused) if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1248,7 +1248,7 @@ time_perf_counter_ns(PyObject *self, PyObject *unused) if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1437,7 +1437,7 @@ time_process_time_ns(PyObject *module, PyObject *unused) if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1610,7 +1610,7 @@ time_thread_time_ns(PyObject *self, PyObject *unused) if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(thread_time_ns_doc, diff --git a/Python/pytime.c b/Python/pytime.c index 0206467364f..2f3d854428b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -2,7 +2,7 @@ #include "pycore_initconfig.h" // _PyStatus_ERR #include "pycore_pystate.h" // _Py_AssertHoldsTstate() #include "pycore_runtime.h" // _PyRuntime -#include "pycore_time.h" // PyTime_t +#include "pycore_time.h" // export _PyLong_FromTime_t() #include // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -472,31 +472,6 @@ _PyTime_FromMicrosecondsClamp(PyTime_t us) } -int -_PyTime_FromLong(PyTime_t *tp, PyObject *obj) -{ - if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_TypeError, "expect int, got %s", - Py_TYPE(obj)->tp_name); - return -1; - } - - static_assert(sizeof(long long) == sizeof(PyTime_t), - "PyTime_t is not long long"); - long long nsec = PyLong_AsLongLong(obj); - if (nsec == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - pytime_overflow(); - } - return -1; - } - - PyTime_t t = (PyTime_t)nsec; - *tp = t; - return 0; -} - - #ifdef HAVE_CLOCK_GETTIME static int pytime_fromtimespec(PyTime_t *tp, const struct timespec *ts, int raise_exc) @@ -658,14 +633,6 @@ PyTime_AsSecondsDouble(PyTime_t ns) } -PyObject * -_PyTime_AsLong(PyTime_t ns) -{ - static_assert(sizeof(long long) >= sizeof(PyTime_t), - "PyTime_t is larger than long long"); - return PyLong_FromLongLong((long long)ns); -} - int _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { From 4ac16dd10950fad2d3e58e8b0ba5f2e621af3cc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 19:00:32 +0100 Subject: [PATCH 029/417] Fix a compiler warning in _randommodule.c (#141058) The test just before the cast ensures that the cast cannot overflow. Fix the warning on 32-bit Windows: Modules\_randommodule.c(525,28): warning C4244: '=': conversion from 'uint64_t' to 'Py_ssize_t', possible loss of data --- Modules/_randommodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index aa2fd28c232..544e636d18f 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -522,7 +522,7 @@ _random_Random_getrandbits_impl(RandomObject *self, uint64_t k) PyErr_NoMemory(); return NULL; } - words = (k - 1u) / 32u + 1u; + words = (Py_ssize_t)((k - 1u) / 32u + 1u); wordarray = (uint32_t *)PyMem_Malloc(words * 4); if (wordarray == NULL) { PyErr_NoMemory(); From baa9f338971c6a13433a8232db77cd45e6b87b77 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Wed, 5 Nov 2025 19:59:59 +0100 Subject: [PATCH 030/417] gh-139313: Improve docs on XML security (GH-139460) Clarify that: - it takes parsing for an attack - that some doors are closed by default - only Expat version 2.7.2 has all the fixes - use of the bundle depends on configuration --- Doc/library/pyexpat.rst | 9 +++++++++ Doc/library/xml.rst | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index f533850c0ca..2f5db81955c 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -634,6 +634,15 @@ otherwise stated. .. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) + .. warning:: + + Implementing a handler that accesses local files and/or the network + may create a vulnerability to + `external entity attacks `_ + if :class:`xmlparser` is used with user-provided XML content. + Please reflect on your `threat model `_ + before implementing this handler. + Called for references to external entities. *base* is the current base, as set by a previous call to :meth:`SetBase`. The public and system identifiers, *systemId* and *publicId*, are strings if given; if the public identifier is not diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 3f745573474..acd8d399fe3 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -53,11 +53,22 @@ XML security An attacker can abuse XML features to carry out denial of service attacks, access local files, generate network connections to other machines, or -circumvent firewalls. +circumvent firewalls when attacker-controlled XML is being parsed, +in Python or elsewhere. -Expat versions lower than 2.6.0 may be vulnerable to "billion laughs", -"quadratic blowup" and "large tokens". Python may be vulnerable if it uses such -older versions of Expat as a system-provided library. +The built-in XML parsers of Python rely on the library `libexpat`_, commonly +called Expat, for parsing XML. + +By default, Expat itself does not access local files or create network +connections. + +Expat versions lower than 2.7.2 may be vulnerable to the "billion laughs", +"quadratic blowup" and "large tokens" vulnerabilities, or to disproportional +use of dynamic memory. +Python bundles a copy of Expat, and whether Python uses the bundled or a +system-wide Expat, depends on how the Python interpreter +:option:`has been configured <--with-system-expat>` in your environment. +Python may be vulnerable if it uses such older versions of Expat. Check :const:`!pyexpat.EXPAT_VERSION`. :mod:`xmlrpc` is **vulnerable** to the "decompression bomb" attack. @@ -90,5 +101,6 @@ large tokens be used to cause denial of service in the application parsing XML. The issue is known as :cve:`2023-52425`. +.. _libexpat: https://github.com/libexpat/libexpat .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb From 3cb1ab0e5de340861afce50f338b2a9d40b04e68 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 5 Nov 2025 22:12:56 +0300 Subject: [PATCH 031/417] gh-131527: Stackref debug borrow checker (#140599) Add borrow checking to the stackref debug mode --------- Co-authored-by: mpage --- Include/internal/pycore_stackref.h | 31 ++++++- ...-10-25-21-31-43.gh-issue-131527.V-JVNP.rst | 2 + Python/stackrefs.c | 91 +++++++++++++++++-- 3 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 15a703a0820..e59611c07fa 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -63,6 +63,8 @@ PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber); PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber); PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber); extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; @@ -248,7 +250,12 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) } else { flags = Py_TAG_REFCNT; } - return _Py_stackref_create(obj, flags, filename, linenumber); + _PyStackRef new_ref = _Py_stackref_create(obj, flags, filename, linenumber); + if (flags == Py_TAG_REFCNT && !_Py_IsImmortal(obj)) { + _PyStackRef borrowed_from = _Py_stackref_get_borrowed_from(ref, filename, linenumber); + _Py_stackref_set_borrowed_from(new_ref, borrowed_from, filename, linenumber); + } + return new_ref; } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) @@ -259,6 +266,7 @@ _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char * assert(!PyStackRef_IsNull(ref)); assert(!PyStackRef_IsTaggedInt(ref)); PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + assert(Py_REFCNT(obj) > 0); if (PyStackRef_RefcountOnObject(ref)) { _Py_DECREF_SPECIALIZED(obj, destruct); } @@ -274,7 +282,11 @@ _PyStackRef_Borrow(_PyStackRef ref, const char *filename, int linenumber) return ref; } PyObject *obj = _Py_stackref_get_object(ref); - return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + _PyStackRef new_ref = _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + if (!_Py_IsImmortal(obj)) { + _Py_stackref_set_borrowed_from(new_ref, ref, filename, linenumber); + } + return new_ref; } #define PyStackRef_Borrow(REF) _PyStackRef_Borrow((REF), __FILE__, __LINE__) @@ -310,13 +322,22 @@ PyStackRef_IsHeapSafe(_PyStackRef ref) static inline _PyStackRef _PyStackRef_MakeHeapSafe(_PyStackRef ref, const char *filename, int linenumber) { - if (PyStackRef_IsHeapSafe(ref)) { + // Special references that can't be closed. + if (ref.index < INITIAL_STACKREF_INDEX) { return ref; } + bool heap_safe = PyStackRef_IsHeapSafe(ref); PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - Py_INCREF(obj); - return _Py_stackref_create(obj, 0, filename, linenumber); + uint16_t flags = 0; + if (heap_safe) { + // Close old ref and create a new one with the same flags. + // This is necessary for correct borrow checking. + flags = ref.index & Py_TAG_BITS; + } else { + Py_INCREF(obj); + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_MakeHeapSafe(REF) _PyStackRef_MakeHeapSafe(REF, __FILE__, __LINE__) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst new file mode 100644 index 00000000000..9969ea058a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst @@ -0,0 +1,2 @@ +Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` +mode. Patch by Mikhail Efimov. diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 720916e0854..0c13cc65510 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -19,6 +19,8 @@ typedef struct _table_entry { int linenumber; const char *filename_borrow; int linenumber_borrow; + int borrows; + _PyStackRef borrowed_from; } TableEntry; TableEntry * @@ -34,6 +36,8 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber) result->linenumber = linenumber; result->filename_borrow = NULL; result->linenumber_borrow = 0; + result->borrows = 0; + result->borrowed_from = PyStackRef_NULL; return result; } @@ -47,11 +51,13 @@ _Py_stackref_get_object(_PyStackRef ref) PyInterpreterState *interp = PyInterpreterState_Get(); assert(interp != NULL); if (ref.index >= interp->next_stackref) { - _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + _Py_FatalErrorFormat(__func__, + "Garbled stack ref with ID %" PRIu64 "\n", ref.index); } TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); if (entry == NULL) { - _Py_FatalErrorFormat(__func__, "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); + _Py_FatalErrorFormat(__func__, + "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); } return entry->obj; } @@ -68,13 +74,16 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) assert(!PyStackRef_IsError(ref)); PyInterpreterState *interp = PyInterpreterState_Get(); if (ref.index >= interp->next_stackref) { - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); - + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); } PyObject *obj; if (ref.index < INITIAL_STACKREF_INDEX) { if (ref.index == 0) { - _Py_FatalErrorFormat(__func__, "Passing NULL to PyStackRef_CLOSE at %s:%d\n", filename, linenumber); + _Py_FatalErrorFormat(__func__, + "Passing NULL to _Py_stackref_close at %s:%d\n", + filename, linenumber); } // Pre-allocated reference to None, False or True -- Do not clear TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); @@ -88,10 +97,27 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) if (entry != NULL) { _Py_FatalErrorFormat(__func__, "Double close of ref ID %" PRIu64 " at %s:%d. Referred to instance of %s at %p. Closed at %s:%d\n", - (void *)ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); + ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); } #endif - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + if (!PyStackRef_IsNull(entry->borrowed_from)) { + _PyStackRef borrowed_from = entry->borrowed_from; + TableEntry *entry_borrowed = _Py_hashtable_get(interp->open_stackrefs_table, (void *)borrowed_from.index); + if (entry_borrowed == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid borrowed StackRef with ID %" PRIu64 " at %s:%d\n", + borrowed_from.index, filename, linenumber); + } + entry_borrowed->borrows--; + } + if (entry->borrows > 0) { + _Py_FatalErrorFormat(__func__, + "StackRef with ID %" PRIu64 " closed with %d borrowed refs at %s:%d. Opened at %s:%d\n", + ref.index, entry->borrows, filename, linenumber, entry->filename, entry->linenumber); } obj = entry->obj; free(entry); @@ -143,15 +169,62 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber if (entry != NULL) { _Py_FatalErrorFormat(__func__, "Borrow of closed ref ID %" PRIu64 " at %s:%d. Referred to instance of %s at %p. Closed at %s:%d\n", - (void *)ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); + ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); } #endif - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); } entry->filename_borrow = filename; entry->linenumber_borrow = linenumber; } +_PyStackRef +_Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + PyInterpreterState *interp = PyInterpreterState_Get(); + + TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + + return entry->borrowed_from; +} + +// This function should be used no more than once per ref. +void +_Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + PyInterpreterState *interp = PyInterpreterState_Get(); + + TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef (ref) with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + + assert(PyStackRef_IsNull(entry->borrowed_from)); + if (PyStackRef_IsNull(borrowed_from)) { + return; + } + + TableEntry *entry_borrowed = _Py_hashtable_get(interp->open_stackrefs_table, (void *)borrowed_from.index); + if (entry_borrowed == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef (borrowed_from) with ID %" PRIu64 " at %s:%d\n", + borrowed_from.index, filename, linenumber); + } + + entry->borrowed_from = borrowed_from; + entry_borrowed->borrows++; +} void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) From 1d25b751c5382aa808dbdfd7eacd77cd793418fc Mon Sep 17 00:00:00 2001 From: Sachin Shah <39803835+inventshah@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:15:27 -0500 Subject: [PATCH 032/417] gh-140650: Fix write(), flush() and close() methods of io.BufferedWriter (GH-140653) They could raise SystemError or crash when getting the "closed" attribute or converting it to boolean raises an exception. --- Lib/test/test_io/test_bufferedio.py | 21 ++++++++++++++++ ...-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst | 3 +++ Modules/_io/bufferedio.c | 25 ++++++++++++++----- 3 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 6e9e96b0e55..30c34e818b1 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -962,6 +962,27 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedWriter"): self.tp(self.BytesIO(), 1024, 1024, 1024) + def test_non_boolean_closed_attr(self): + # gh-140650: check TypeError is raised + class MockRawIOWithoutClosed(self.MockRawIO): + closed = NotImplemented + + bufio = self.tp(MockRawIOWithoutClosed()) + self.assertRaises(TypeError, bufio.write, b"") + self.assertRaises(TypeError, bufio.flush) + self.assertRaises(TypeError, bufio.close) + + def test_closed_attr_raises(self): + class MockRawIOClosedRaises(self.MockRawIO): + @property + def closed(self): + raise ValueError("test") + + bufio = self.tp(MockRawIOClosedRaises()) + self.assertRaisesRegex(ValueError, "test", bufio.write, b"") + self.assertRaisesRegex(ValueError, "test", bufio.flush) + self.assertRaisesRegex(ValueError, "test", bufio.close) + class PyBufferedWriterTest(BufferedWriterTest, PyTestCase): tp = pyio.BufferedWriter diff --git a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst new file mode 100644 index 00000000000..2ae153a6480 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst @@ -0,0 +1,3 @@ +Fix an issue where closing :class:`io.BufferedWriter` could crash if +the closed attribute raised an exception on access or could not be +converted to a boolean. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 0a2b3502532..0b4bc4c6b8a 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -362,16 +362,24 @@ _enter_buffered_busy(buffered *self) } #define IS_CLOSED(self) \ - (!self->buffer || \ + (!self->buffer ? 1 : \ (self->fast_closed_checks \ ? _PyFileIO_closed(self->raw) \ : buffered_closed(self))) #define CHECK_CLOSED(self, error_msg) \ - if (IS_CLOSED(self) && (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) { \ - PyErr_SetString(PyExc_ValueError, error_msg); \ - return NULL; \ - } \ + do { \ + int _closed = IS_CLOSED(self); \ + if (_closed < 0) { \ + return NULL; \ + } \ + if (_closed && \ + (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) \ + { \ + PyErr_SetString(PyExc_ValueError, error_msg); \ + return NULL; \ + } \ + } while (0); #define VALID_READ_BUFFER(self) \ (self->readable && self->read_end != -1) @@ -2079,6 +2087,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) PyObject *res = NULL; Py_ssize_t written, avail, remaining; Py_off_t offset; + int r; CHECK_INITIALIZED(self) @@ -2087,7 +2096,11 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) /* Issue #31976: Check for closed file after acquiring the lock. Another thread could be holding the lock while closing the file. */ - if (IS_CLOSED(self)) { + r = IS_CLOSED(self); + if (r < 0) { + goto error; + } + if (r > 0) { PyErr_SetString(PyExc_ValueError, "write to closed file"); goto error; } From f458ac01ba522cc7f94c0c0ee9a00c82f1be6d69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 20:18:45 +0100 Subject: [PATCH 033/417] Fix compiler warnings in remote debugging (#141060) Example of fixed warnings on 32-bit Windows: Python\remote_debugging.c(24,53): warning C4244: 'function': conversion from 'uint64_t' to 'uintptr_t', possible loss of data Modules\_remote_debugging_module.c(789,44): warning C4244: 'function': conversion from 'uint64_t' to 'size_t', possible loss of data --- Modules/_remote_debugging_module.c | 58 +++++++++++++++--------------- Python/remote_debugging.c | 14 ++++---- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 5937d4892f5..c6ced39c70c 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -503,7 +503,7 @@ iterate_threads( if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - unwinder->interpreter_addr + unwinder->debug_offsets.interpreter_state.threads_main, + unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main, sizeof(void*), &thread_state_addr)) { @@ -514,7 +514,7 @@ iterate_threads( while (thread_state_addr != 0) { if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - thread_state_addr + unwinder->debug_offsets.thread_state.native_thread_id, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.native_thread_id, sizeof(tid), &tid)) { @@ -530,7 +530,7 @@ iterate_threads( // Move to next thread if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - thread_state_addr + unwinder->debug_offsets.thread_state.next, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.next, sizeof(void*), &thread_state_addr)) { @@ -686,7 +686,7 @@ read_py_str( return NULL; } - size_t offset = unwinder->debug_offsets.unicode_object.asciiobject_size; + size_t offset = (size_t)unwinder->debug_offsets.unicode_object.asciiobject_size; res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); if (res < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read string data from remote memory"); @@ -748,7 +748,7 @@ read_py_bytes( return NULL; } - size_t offset = unwinder->debug_offsets.bytes_object.ob_sval; + size_t offset = (size_t)unwinder->debug_offsets.bytes_object.ob_sval; res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); if (res < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read bytes data from remote memory"); @@ -786,7 +786,7 @@ read_py_long( int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, address, - unwinder->debug_offsets.long_object.size, + (size_t)unwinder->debug_offsets.long_object.size, long_obj); if (bytes_read < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyLongObject"); @@ -823,7 +823,7 @@ read_py_long( bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - address + unwinder->debug_offsets.long_object.ob_digit, + address + (uintptr_t)unwinder->debug_offsets.long_object.ob_digit, sizeof(digit) * size, digits ); @@ -933,7 +933,7 @@ parse_task_name( int err = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, task_obj); if (err < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); @@ -1040,7 +1040,7 @@ handle_yield_from_frame( uintptr_t gi_await_addr_type_addr; err = read_ptr( unwinder, - gi_await_addr + unwinder->debug_offsets.pyobject.ob_type, + gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type, &gi_await_addr_type_addr); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address"); @@ -1101,7 +1101,7 @@ parse_coro_chain( // Parse the previous frame using the gi_iframe from local copy uintptr_t prev_frame; - uintptr_t gi_iframe_addr = coro_address + unwinder->debug_offsets.gen_object.gi_iframe; + uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; uintptr_t address_of_code_object = 0; if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); @@ -1153,7 +1153,7 @@ create_task_result( // Parse coroutine chain if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, task_obj) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object for coro chain"); goto error; @@ -1206,7 +1206,7 @@ parse_task( err = read_char( unwinder, - task_address + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + task_address + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_is_task, &is_task); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read is_task flag"); @@ -1354,7 +1354,7 @@ process_thread_for_awaited_by( void *context ) { PyObject *result = (PyObject *)context; - uintptr_t head_addr = thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + uintptr_t head_addr = thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; return append_awaited_by(unwinder, tid, head_addr, result); } @@ -1369,7 +1369,7 @@ process_task_awaited_by( // Read the entire TaskObj at once char task_obj[SIZEOF_TASK_OBJ]; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, task_obj) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); return -1; @@ -1526,7 +1526,7 @@ find_running_task_in_thread( uintptr_t address_of_running_loop; int bytes_read = read_py_ptr( unwinder, - thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, &address_of_running_loop); if (bytes_read == -1) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); @@ -1540,7 +1540,7 @@ find_running_task_in_thread( int err = read_ptr( unwinder, - thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, running_task_addr); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); @@ -1556,7 +1556,7 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt if(read_py_ptr( unwinder, - task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_coro, + task_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_coro, &running_coro_addr) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); return -1; @@ -1572,7 +1572,7 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt // the offset leads directly to its first field: f_executable if (read_py_ptr( unwinder, - running_coro_addr + unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { + running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); return -1; } @@ -1741,7 +1741,7 @@ static bool parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) { const uint8_t* ptr = (const uint8_t*)(linetable); - uint64_t addr = 0; + uintptr_t addr = 0; info->lineno = firstlineno; while (*ptr != '\0') { @@ -1870,7 +1870,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->file_name = file; meta->linetable = linetable; meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); - meta->addr_code_adaptive = real_address + unwinder->debug_offsets.code_object.co_code_adaptive; + meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; if (unwinder && unwinder->code_object_cache && _Py_hashtable_set(unwinder->code_object_cache, key, meta) < 0) { cached_code_metadata_destroy(meta); @@ -2037,7 +2037,7 @@ copy_stack_chunks(RemoteUnwinderObject *unwinder, size_t count = 0; size_t max_chunks = 16; - if (read_ptr(unwinder, tstate_addr + unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { + if (read_ptr(unwinder, tstate_addr + (uintptr_t)unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read initial stack chunk address"); return -1; } @@ -2146,8 +2146,8 @@ populate_initial_state_data( uintptr_t *interpreter_state, uintptr_t *tstate ) { - uint64_t interpreter_state_list_head = - unwinder->debug_offsets.runtime_state.interpreters_head; + uintptr_t interpreter_state_list_head = + (uintptr_t)unwinder->debug_offsets.runtime_state.interpreters_head; uintptr_t address_of_interpreter_state; int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( @@ -2174,7 +2174,7 @@ populate_initial_state_data( } uintptr_t address_of_thread = address_of_interpreter_state + - unwinder->debug_offsets.interpreter_state.threads_main; + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main; if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, @@ -2198,7 +2198,7 @@ find_running_frame( if ((void*)address_of_thread != NULL) { int err = read_ptr( unwinder, - address_of_thread + unwinder->debug_offsets.thread_state.current_frame, + address_of_thread + (uintptr_t)unwinder->debug_offsets.thread_state.current_frame, frame); if (err) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read current frame pointer"); @@ -2370,7 +2370,7 @@ append_awaited_by_for_thread( } uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) - - unwinder->async_debug_offsets.asyncio_task_object.task_node; + - (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_node; if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process task node in awaited_by"); @@ -2605,7 +2605,7 @@ get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread } SYSTEM_THREAD_INFORMATION *ti = (SYSTEM_THREAD_INFORMATION *)((char *)pi + sizeof(SYSTEM_PROCESS_INFORMATION)); - for (Py_ssize_t i = 0; i < pi->NumberOfThreads; i++, ti++) { + for (size_t i = 0; i < pi->NumberOfThreads; i++, ti++) { if (ti->ClientId.UniqueThread == (HANDLE)tid) { return ti->ThreadState != WIN32_THREADSTATE_RUNNING ? THREAD_STATE_IDLE : THREAD_STATE_RUNNING; } @@ -2642,7 +2642,7 @@ unwind_stack_for_thread( char ts[SIZEOF_THREAD_STATE]; int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, *current_tstate, unwinder->debug_offsets.thread_state.size, ts); + &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); if (bytes_read < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); goto error; @@ -3174,7 +3174,7 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s } uintptr_t head_addr = self->interpreter_addr - + self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; + + (uintptr_t)self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; // On top of a per-thread task lists used by default by asyncio to avoid // contention, there is also a fallback per-interpreter list of tasks; diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index 7aee87ef05a..71ffb17ed68 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -19,7 +19,7 @@ cleanup_proc_handle(proc_handle_t *handle) { } static int -read_memory(proc_handle_t *handle, uint64_t remote_address, size_t len, void* dst) +read_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } @@ -235,7 +235,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc int is_remote_debugging_enabled = 0; if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.debugger_support.remote_debugging_enabled, + interpreter_state_addr + (uintptr_t)debug_offsets.debugger_support.remote_debugging_enabled, sizeof(int), &is_remote_debugging_enabled)) { @@ -255,7 +255,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (tid != 0) { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_head, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_head, sizeof(void*), &thread_state_addr)) { @@ -264,7 +264,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc while (thread_state_addr != 0) { if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.native_thread_id, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.native_thread_id, sizeof(this_tid), &this_tid)) { @@ -277,7 +277,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.next, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.next, sizeof(void*), &thread_state_addr)) { @@ -294,7 +294,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc } else { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_main, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_main, sizeof(void*), &thread_state_addr)) { @@ -346,7 +346,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc uintptr_t eval_breaker; if (0 != read_memory( handle, - thread_state_addr + debug_offsets.debugger_support.eval_breaker, + thread_state_addr + (uintptr_t)debug_offsets.debugger_support.eval_breaker, sizeof(uintptr_t), &eval_breaker)) { From 986bb0a1a2bd290f5da347e455b23468aa3f62f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Nov 2025 21:16:37 +0100 Subject: [PATCH 034/417] gh-83714: Fix stat_nanosecond_timestamp() for 32-bit time_t (#141069) --- Modules/posixmodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ecda75ec6ab..6390f1fc5fe 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2634,13 +2634,14 @@ _posix_free(void *module) static PyObject * stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) { -#if SIZEOF_LONG >= 8 +#if SIZEOF_TIME_T == 4 + return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); +#else /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); } else -#endif { PyObject *ns_total = NULL; PyObject *s_in_ns = NULL; @@ -2663,6 +2664,7 @@ stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) Py_XDECREF(s_in_ns); return ns_total; } +#endif } static int From b83f379a972c001864d3593cd64fc07e7c7f375f Mon Sep 17 00:00:00 2001 From: Edward Xu Date: Thu, 6 Nov 2025 05:20:40 +0800 Subject: [PATCH 035/417] gh-133467: Fix typeobject `tp_base` race in free threading (gh-140549) --- Lib/test/test_free_threading/test_type.py | 19 +++++++++++++++++++ ...-10-24-14-29-12.gh-issue-133467.A5d6TM.rst | 1 + Objects/typeobject.c | 10 ++++++++++ Tools/tsan/suppressions_free_threading.txt | 4 ---- 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 2d995751005..1255d842dbf 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -141,6 +141,25 @@ def reader(): self.run_one(writer, reader) + def test_bases_change(self): + class BaseA: + pass + + class Derived(BaseA): + pass + + def writer(): + for _ in range(1000): + class BaseB: + pass + Derived.__bases__ = (BaseB,) + + def reader(): + for _ in range(1000): + Derived.__base__ + + self.run_one(writer, reader) + def run_one(self, writer_func, reader_func): barrier = threading.Barrier(NTHREADS) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst new file mode 100644 index 00000000000..f69786866e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst @@ -0,0 +1 @@ +Fix race when updating :attr:`!type.__bases__` that could allow a read of :attr:`!type.__base__` to observe an inconsistent value on the free threaded build. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 326f4add896..58228d62485 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -189,6 +189,8 @@ type_lock_allow_release(void) #define types_world_is_stopped() 1 #define types_stop_world() #define types_start_world() +#define type_lock_prevent_release() +#define type_lock_allow_release() #endif @@ -1920,8 +1922,12 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, PyTypeObject *b assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; + type_lock_prevent_release(); + types_stop_world(); set_tp_bases(type, Py_NewRef(new_bases), 0); type->tp_base = (PyTypeObject *)Py_NewRef(best_base); + types_start_world(); + type_lock_allow_release(); PyObject *temp = PyList_New(0); if (temp == NULL) { @@ -1982,8 +1988,12 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, PyTypeObject *b if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == best_base); + type_lock_prevent_release(); + types_stop_world(); set_tp_bases(type, old_bases, 0); type->tp_base = old_base; + types_start_world(); + type_lock_allow_release(); Py_DECREF(new_bases); Py_DECREF(best_base); diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 6bd31e8e6ec..404c3015736 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -41,7 +41,3 @@ race:list_inplace_repeat_lock_held # PyObject_Realloc internally does memcpy which isn't atomic so can race # with non-locking reads. See #132070 race:PyObject_Realloc - -# gh-133467. Some of these could be hard to trigger. -race_top:set_tp_bases -race_top:type_set_bases_unlocked From 11fc411f98a04947a2a21329c29fe0f35ff52dba Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 6 Nov 2025 06:49:45 +0900 Subject: [PATCH 036/417] gh-140916: Remove unused codes in winreg.c (#140934) --- PC/winreg.c | 55 ----------------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/PC/winreg.c b/PC/winreg.c index c7bc74728f1..3cc6123fc3a 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -425,19 +425,6 @@ static PyType_Spec pyhkey_type_spec = { /************************************************************************ The public PyHKEY API (well, not public yet :-) ************************************************************************/ -PyObject * -PyHKEY_New(PyObject *m, HKEY hInit) -{ - winreg_state *st = _PyModule_GetState(m); - PyHKEYObject *key = PyObject_GC_New(PyHKEYObject, st->PyHKEY_Type); - if (key == NULL) { - return NULL; - } - key->hkey = hInit; - PyObject_GC_Track(key); - return (PyObject *)key; -} - BOOL PyHKEY_Close(winreg_state *st, PyObject *ob_handle) { @@ -513,48 +500,6 @@ PyHKEY_FromHKEY(winreg_state *st, HKEY h) } -/************************************************************************ - The module methods -************************************************************************/ -BOOL -PyWinObject_CloseHKEY(winreg_state *st, PyObject *obHandle) -{ - BOOL ok; - if (PyHKEY_Check(st, obHandle)) { - ok = PyHKEY_Close(st, obHandle); - } -#if SIZEOF_LONG >= SIZEOF_HKEY - else if (PyLong_Check(obHandle)) { - long rc; - Py_BEGIN_ALLOW_THREADS - rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); - Py_END_ALLOW_THREADS - ok = (rc == ERROR_SUCCESS); - if (!ok) - PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); - } -#else - else if (PyLong_Check(obHandle)) { - long rc; - HKEY hkey = (HKEY)PyLong_AsVoidPtr(obHandle); - Py_BEGIN_ALLOW_THREADS - rc = RegCloseKey(hkey); - Py_END_ALLOW_THREADS - ok = (rc == ERROR_SUCCESS); - if (!ok) - PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); - } -#endif - else { - PyErr_SetString( - PyExc_TypeError, - "A handle must be a HKEY object or an integer"); - return FALSE; - } - return ok; -} - - /* Private Helper functions for the registry interfaces From 5b02c6e920aaef4b202fc19186f742d008460fd3 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 17:00:26 -0500 Subject: [PATCH 037/417] gh-141004: Document `Py_RETURN_NAN` and `Py_RETURN_INF` (GH-141029) Co-authored-by: Sergey B Kirpichev --- Doc/c-api/float.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 489676caa3a..1085c32a537 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,23 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + Pack and Unpack functions ------------------------- From 227f4abacdd89bb3816c172a7f6fdaa2024dbada Mon Sep 17 00:00:00 2001 From: Zenith Date: Wed, 5 Nov 2025 17:00:36 -0500 Subject: [PATCH 038/417] gh-76007: remove curses.__version__ doc (#141052) --- Doc/library/curses.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index fb84cf32246..e60197ddd89 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -1349,7 +1349,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. From f0ab07f22c5fd18058a3ece7a1e745b3922af908 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 5 Nov 2025 17:32:12 -0500 Subject: [PATCH 039/417] gh-141004: Document `PyDict_GET_SIZE` (GH-141078) --- Doc/c-api/dict.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0abbd662dad..246ce5391e1 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -245,6 +245,11 @@ Dictionary Objects ``len(p)`` on a dictionary. +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) Iterate over all key-value pairs in the dictionary *p*. The From 95f6e1275b1c9de550d978cb2b4351cc4ed24fe4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 5 Nov 2025 14:46:30 -0800 Subject: [PATCH 040/417] GH-108009: Add clarification of parser and argument defaults in argparse docs (#124154) Co-authored-by: C.A.M. Gerlach --- Doc/library/argparse.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 9655db4f301..5a8f0bde2e3 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -2070,7 +2070,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') From 101c9c0a2187940900f684086cb9ba0d456fda49 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Thu, 6 Nov 2025 10:21:13 +0530 Subject: [PATCH 041/417] gh-118516: clarify that subprocess are automatically killed if transport gets garbage collected (#140997) --- Doc/library/asyncio-eventloop.rst | 6 ++++++ Doc/library/asyncio-subprocess.rst | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 0ccc7a2b448..72f484fd1cb 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1631,6 +1631,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1654,6 +1657,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 03e76bc8689..9416c758e51 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -76,6 +76,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -95,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and From 8822166200ddb4a7635337b97b626e658a443cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20S=C5=82awecki?= Date: Thu, 6 Nov 2025 06:39:07 +0100 Subject: [PATCH 042/417] gh-140569: recommend the new REPL in the asyncio REPL docs (#140570) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/asyncio.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 444db01390d..0f72e31dee5 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -79,6 +79,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) From 9037a386c6ed0c71cf8525ef91c55694ebeedc36 Mon Sep 17 00:00:00 2001 From: RayXu <140802139+F18-Maverick@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:48:30 +0800 Subject: [PATCH 043/417] docs: fix a grammatical error in function.rst (#140990) --- Doc/c-api/function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 764b2ac610b..0bac6389571 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -200,7 +200,7 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + If *event* is ``PyFunction_EVENT_DESTROY``, taking a reference in the callback to the about-to-be-destroyed function will resurrect it, preventing it from being freed at this time. When the resurrected object is destroyed later, any watcher callbacks active at that time will be called again. From d6c89a2df2c8b7603125883494e9058a88348f66 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 6 Nov 2025 05:50:57 +0000 Subject: [PATCH 044/417] gh-140939: Fix memory leak in `_PyBytes_FormatEx` error path (#140957) --- Lib/test/test_bytes.py | 7 +++++++ .../2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst | 2 ++ Objects/bytesobject.c | 1 + 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f10e4041937..e012042159d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -802,6 +802,13 @@ def __int__(self): with self.assertRaisesRegex(TypeError, msg): operator.mod(format_bytes, value) + def test_memory_leak_gh_140939(self): + # gh-140939: MemoryError is raised without leaking + _testcapi = import_helper.import_module('_testcapi') + with self.assertRaises(MemoryError): + b = self.type2test(b'%*b') + b % (_testcapi.PY_SSIZE_T_MAX, b'abc') + def test_imod(self): b = self.type2test(b'hello, %b!') orig = b diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst new file mode 100644 index 00000000000..a2921761f75 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst @@ -0,0 +1,2 @@ +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with the +``%*b`` format with a large width that results in a :exc:`MemoryError`. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 9c807b3dd16..2b9513abe91 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -985,6 +985,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, if (alloc > 2) { res = PyBytesWriter_GrowAndUpdatePointer(writer, alloc - 2, res); if (res == NULL) { + Py_XDECREF(temp); goto error; } } From 6a7c969d003d3ba932d5c7f14a58e2a6408f4a3d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Thu, 6 Nov 2025 03:08:24 -0500 Subject: [PATCH 045/417] gh-129876: Move misplaced IDLE news item (#141118) --- Lib/idlelib/News3.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 30784578cc6..53d83762f99 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,9 @@ Released on 2025-10-07 ========================= +gh-129873: Simplify displaying the IDLE doc by only copying the text +section of idle.html to idlelib/help.html. Patch by Stan Ulbrych. + gh-112936: IDLE - Include Shell menu in single-process mode, though with Restart Shell and View Last Restart disabled. Patch by Zhikang Yan. @@ -26,9 +29,6 @@ Released on 2024-10-07 gh-120104: Fix padding in config and search dialog windows in IDLE. -gh-129873: Simplify displaying the IDLE doc by only copying the text -section of idle.html to idlelib/help.html. Patch by Stan Ulbrych. - gh-120083: Add explicit black IDLE Hovertip foreground color needed for recent macOS. Fixes Sonoma showing unreadable white on pale yellow. Patch by John Riggles. From 4e6e208be9d1c52d1b55a8bb3a83682cb078e55e Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:21:02 +0000 Subject: [PATCH 046/417] Minor fixes to `idle.rst` and regenerate `help.html` (#140037) --- .gitattributes | 1 + Doc/library/idle.rst | 6 +-- Lib/idlelib/help.html | 110 ++++++++++++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 35 deletions(-) diff --git a/.gitattributes b/.gitattributes index 823e3e975a2..d6547212393 100644 --- a/.gitattributes +++ b/.gitattributes @@ -83,6 +83,7 @@ Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/idlelib/help.html generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index e547c96b580..10ec7f0a6f1 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -13,7 +13,7 @@ IDLE --- Python editor and shell single: Integrated Development Environment .. - Remember to update Lib/idlelib/help.html with idlelib.help.copy_source() when modifying this file. + Remember to update Lib/idlelib/help.html with idlelib.help.copy_strip() when modifying this file. -------------- @@ -88,7 +88,7 @@ Save Save As... Save the current window with a Save As dialog. The file saved becomes the - new associated file for the window. (If your file namager is set to hide + new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no '.', '.py' and '.txt' will be added for Python and text files, except that on macOS Aqua,'.py' is added for all files.) @@ -206,7 +206,7 @@ New Indent Width Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last - non-whitespace character of a line by applying str.rstrip to each line, + non-whitespace character of a line by applying :meth:`str.rstrip` to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file. diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ebff9a309d9..fc618ab727d 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -53,7 +53,7 @@ and after the window title. If there is no associated file, do Save As instead.

Save As…

Save the current window with a Save As dialog. The file saved becomes the -new associated file for the window. (If your file namager is set to hide +new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no ‘.’, ‘.py’ and ‘.txt’ will be added for Python and text files, except that on macOS Aqua,’.py’ is added for all files.)

@@ -143,8 +143,8 @@ paragraph will be formatted to less than N columns, where N defaults to 72.

New Indent Width

Open a dialog to change indent width. The accepted default by the Python community is 4 spaces.

-
Strip Trailing Chitespace

Remove trailing space and other whitespace characters after the last -non-whitespace character of a line by applying str.rstrip to each line, +

Strip Trailing Whitespace

Remove trailing space and other whitespace characters after the last +non-whitespace character of a line by applying str.rstrip() to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file.

@@ -337,16 +337,16 @@ Unix and the Command key on assume that the keys have not been re-bound to something else.)

  • Arrow keys move the cursor one character or line.

    • -

  • C-LeftArrow and C-RightArrow moves left or right one word.

    • +

  • C-LeftArrow and C-RightArrow moves left or right one word.

  • Home and End go to the beginning or end of the line.

  • Page Up and Page Down go up or down one screen.

    • -

  • C-Home and C-End go to beginning or end of the file.

    • -

  • Backspace and Del (or C-d) delete the previous +

  • C-Home and C-End go to beginning or end of the file.

    • +

  • Backspace and Del (or C-d) delete the previous or next character.

    • -

  • C-Backspace and C-Del delete one word left or right.

    • -

  • C-k deletes (‘kills’) everything to the right.

    • +

  • C-Backspace and C-Del delete one word left or right.

    • +

  • C-k deletes (‘kills’) everything to the right.

-

Standard keybindings (like C-c to copy and C-v to paste) +

Standard keybindings (like C-c to copy and C-v to paste) may work. Keybindings are selected in the Configure IDLE dialog.

@@ -390,7 +390,7 @@ one can specify a drive first.) Move into subdirectories by typing a directory name and a separator.

Instead of waiting, or after a box is closed, open a completion box immediately with Show Completions on the Edit menu. The default hot -key is C-space. If one types a prefix for the desired name +key is C-space. If one types a prefix for the desired name before opening the box, the first match or near miss is made visible. The result is the same as if one enters a prefix after the box is displayed. Show Completions after a quote completes @@ -473,9 +473,9 @@ in an editor window.

The editing features described in previous subsections work when entering code interactively. IDLE’s Shell window also responds to the following:

    -

  • C-c attempts to interrupt statement execution (but may fail).

    • -

  • C-d closes Shell if typed at a >>> prompt.

    • -

  • Alt-p and Alt-n (C-p and C-n on macOS) +

  • C-c attempts to interrupt statement execution (but may fail).

    • +

  • C-d closes Shell if typed at a >>> prompt.

    • +

  • Alt-p and Alt-n (C-p and C-n on macOS) retrieve to the current prompt the previous or next previously entered statement that matches anything already typed.

  • Return while the cursor is on any previous statement @@ -517,27 +517,73 @@ executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE’s Python shell.

    Command line usage

    -
    idle.py [-c command] [-d] [-e] [-h] [-i] [-r file] [-s] [-t title] [-] [arg] ...
-
--c command  run command in the shell window
--d          enable debugger and open shell window
--e          open editor window
--h          print help message with legal combinations and exit
--i          open shell window
--r file     run file in shell window
--s          run $IDLESTARTUP or $PYTHONSTARTUP first, in shell window
--t title    set title of shell window
--           run stdin in shell (- must be last option before args)
+
    IDLE can be invoked from the command line with various options. The general syntax is:
    
+
    python -m idlelib [options] [file ...]
 
    
 
    
-
    If there are arguments:
    
+
    The following options are available:
    
+
    
+
    
+-c <command>
    
+
    Run the specified Python command in the shell window.
+For example, pass -c "print('Hello, World!')".
+On Windows, the outer quotes must be double quotes as shown.
    
+
    
+
+
    
+
    
+-d
    
+
    Enable the debugger and open the shell window.
    
+
    
+
+
    
+
    
+-e
    
+
    Open an editor window.
    
+
    
+
+
    
+
    
+-h
    
+
    Print a help message with legal combinations of options and exit.
    
+
    
+
+
    
+
    
+-i
    
+
    Open a shell window.
    
+
    
+
+
    
+
    
+-r <file>
    
+
    Run the specified file in the shell window.
    
+
    
+
+
    
+
    
+-s
    
+
    Run the startup file (as defined by the environment variables IDLESTARTUP or PYTHONSTARTUP) before opening the shell window.
    
+
    
+
+
    
+
    
+-t <title>
    
+
    Set the title of the shell window.
    
+
    
+
+
    
+
    
+-
    
+
    Read and execute standard input in the shell window. This option must be the last one before any arguments.
    
+
    
+
+
    If arguments are provided:
    
 
      
-
    • If -, -c, or r is used, all arguments are placed in
-sys.argv[1:...] and sys.argv[0] is set to '', '-c',
-or '-r'.  No editor window is opened, even if that is the default
-set in the Options dialog.
      • 
-
    • Otherwise, arguments are files opened for editing and
-sys.argv reflects the arguments passed to IDLE itself.
      • 
+
    • If -, -c, or -r is used, all arguments are placed in sys.argv[1:],
+and sys.argv[0] is set to '', '-c', or '-r' respectively.
+No editor window is opened, even if that is the default set in the Options dialog.
      • 
+
    • Otherwise, arguments are treated as files to be opened for editing, and sys.argv reflects the arguments passed to IDLE itself.
      •
    @@ -798,7 +844,7 @@ of this page for how to use IDLE.

    either in idlelib or click Help => About IDLE on the IDLE menu. This file also maps IDLE menu items to the code that implements the item. Except for files listed under ‘Startup’, the idlelib code is ‘private’ in -sense that feature changes can be backported (see PEP 434).

    +sense that feature changes can be backported (see PEP 434).

From 13360efd385d1a7d0659beba03787ea3d063ef9b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Nov 2025 11:34:32 +0200 Subject: [PATCH 047/417] gh-125346: Add more base64 tests (GH-141061) Add more tests for the altchars argument of b64decode() and for the map01 argument of b32decode(). --- Lib/test/test_base64.py | 64 ++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 6b5c65a56d8..65977ca8c9f 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -231,18 +231,6 @@ def test_b64decode(self): self.check_other_types(base64.b64decode, b"YWJj", b"abc") self.check_decode_type_errors(base64.b64decode) - # Test with arbitrary alternative characters - tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', - } - for (data, altchars), res in tests_altchars.items(): - data_str = data.decode('ascii') - altchars_str = altchars.decode('ascii') - - eq(base64.b64decode(data, altchars=altchars), res) - eq(base64.b64decode(data_str, altchars=altchars), res) - eq(base64.b64decode(data, altchars=altchars_str), res) - eq(base64.b64decode(data_str, altchars=altchars_str), res) - # Test standard alphabet for data, res in tests.items(): eq(base64.standard_b64decode(data), res) @@ -263,6 +251,20 @@ def test_b64decode(self): b'\xd3V\xbeo\xf7\x1d') self.check_decode_type_errors(base64.urlsafe_b64decode) + def test_b64decode_altchars(self): + # Test with arbitrary alternative characters + eq = self.assertEqual + res = b'\xd3V\xbeo\xf7\x1d' + for altchars in b'*$', b'+/', b'/+', b'+_', b'-+', b'-/', b'/_': + data = b'01a%cb%ccd' % tuple(altchars) + data_str = data.decode('ascii') + altchars_str = altchars.decode('ascii') + + eq(base64.b64decode(data, altchars=altchars), res) + eq(base64.b64decode(data_str, altchars=altchars), res) + eq(base64.b64decode(data, altchars=altchars_str), res) + eq(base64.b64decode(data_str, altchars=altchars_str), res) + def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') @@ -295,10 +297,12 @@ def test_b64decode_invalid_chars(self): base64.b64decode(bstr.decode('ascii'), validate=True) # Normal alphabet characters not discarded when alternative given - res = b'\xFB\xEF\xBE\xFF\xFF\xFF' - self.assertEqual(base64.b64decode(b'++[[//]]', b'[]'), res) - self.assertEqual(base64.urlsafe_b64decode(b'++--//__'), res) - + res = b'\xfb\xef\xff' + self.assertEqual(base64.b64decode(b'++//', validate=True), res) + self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) + self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) + self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) + self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) def _altchars_strategy(): """Generate 'altchars' for base64 encoding.""" @@ -394,19 +398,33 @@ def test_b32decode_casefold(self): eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe', - (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe', - } - for (data, map01), res in map_tests.items(): - data_str = data.decode('ascii') + def test_b32decode_map01(self): + # Mapping zero and one + eq = self.assertEqual + res_L = b'b\xdd\xad\xf3\xbe' + res_I = b'b\x1d\xad\xf3\xbe' + eq(base64.b32decode(b'MLO23456'), res_L) + eq(base64.b32decode('MLO23456'), res_L) + eq(base64.b32decode(b'MIO23456'), res_I) + eq(base64.b32decode('MIO23456'), res_I) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456') + self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') + self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') + self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') + + data = b'M1023456' + data_str = data.decode('ascii') + for map01, res in [(b'L', res_L), (b'I', res_I)]: map01_str = map01.decode('ascii') eq(base64.b32decode(data, map01=map01), res) eq(base64.b32decode(data_str, map01=map01), res) eq(base64.b32decode(data, map01=map01_str), res) eq(base64.b32decode(data_str, map01=map01_str), res) - self.assertRaises(binascii.Error, base64.b32decode, data) - self.assertRaises(binascii.Error, base64.b32decode, data_str) + + eq(base64.b32decode(b'M1O23456', map01=map01), res) + eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) + eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] From 86ab7bb87a3b8c7d617763bffc1992791c0e9bde Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 6 Nov 2025 11:20:02 +0100 Subject: [PATCH 048/417] gh-137232: Update free-threading HOWTOs with up-to-date info for 3.14 (#140817) --- Doc/howto/free-threading-extensions.rst | 11 ++++++----- Doc/howto/free-threading-python.rst | 21 ++++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 3776132c685..5647ab2d87c 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -203,7 +203,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -344,12 +344,12 @@ This means you cannot rely on nested critical sections to lock multiple objects at once, as the inner critical section may suspend the outer ones. Instead, use :c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. -Note that the locks described above are only :c:type:`!PyMutex` based locks. +Note that the locks described above are only :c:type:`PyMutex` based locks. The critical section implementation does not know about or affect other locking mechanisms that might be in use, like POSIX mutexes. Also note that while -blocking on any :c:type:`!PyMutex` causes the critical sections to be +blocking on any :c:type:`PyMutex` causes the critical sections to be suspended, only the mutexes that are part of the critical sections are -released. If :c:type:`!PyMutex` is used without a critical section, it will +released. If :c:type:`PyMutex` is used without a critical section, it will not be released and therefore does not get the same deadlock avoidance. Important Considerations @@ -397,7 +397,8 @@ The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the free-threaded build, with the ``t`` suffix, such as ``python3.13t``. * `pypa/cibuildwheel `_ supports the - free-threaded build if you set + free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded + wheels will be built by default. On Python 3.13, you will need to set `CIBW_ENABLE to cpython-freethreading `_. Limited C API and Stable ABI diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 24069617c47..e4df7a787a2 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -116,12 +116,14 @@ after the main thread is running. The following objects are immortalized: * :ref:`classes ` (type objects) Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. +objects of these types may see increased memory usage under Python 3.13. This +has been addressed in the 3.14 release, where the aforementioned objects use +deferred reference counting to avoid reference count contention. Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +returned by :func:`sys.intern` are also immortalized in the 3.13 release. This +behavior is part of the 3.14 release as well and it is expected to remain in +future free-threaded builds. Frame objects @@ -150,11 +152,12 @@ compared to the default GIL-enabled build. In 3.13, this overhead is about 40% on the `pyperformance `_ suite. Programs that spend most of their time in C extensions or I/O will see less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +interpreter (:pep:`659`) is disabled in the free-threaded build. + +The specializing adaptive interpreter has been re-enabled in a thread-safe way +in the 3.14 release. The performance penalty on single-threaded code in +free-threaded mode is now roughly 5-10%, depending on the platform and C +compiler used. Behavioral changes From d2ce6d708a9eaac4e546744ca4da359ee6901ebc Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 6 Nov 2025 11:49:44 +0100 Subject: [PATCH 049/417] gh-139707: Add docs for optional modules (GH-140171) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Emma Smith Co-authored-by: Author: Terry Jan Reedy Co-authored-by: Victor Stinner --- Doc/glossary.rst | 9 ++ Doc/includes/optional-module.rst | 9 ++ Doc/library/bz2.rst | 2 + Doc/library/compression.zstd.rst | 2 + Doc/library/ctypes.rst | 2 + Doc/library/curses.rst | 2 + Doc/library/ensurepip.rst | 2 + Doc/library/gzip.rst | 2 + Doc/library/idle.rst | 4 + Doc/library/lzma.rst | 2 + Doc/library/readline.rst | 2 + Doc/library/sqlite3.rst | 4 +- Doc/library/ssl.rst | 5 +- Doc/library/tarfile.rst | 8 ++ Doc/library/tkinter.rst | 2 + Doc/library/turtle.rst | 2 + Doc/library/zipfile.rst | 10 ++ Doc/library/zlib.rst | 6 +- Doc/using/configure.rst | 216 ++++++++++++++++++++----------- Doc/using/unix.rst | 12 +- 20 files changed, 221 insertions(+), 82 deletions(-) create mode 100644 Doc/includes/optional-module.rst diff --git a/Doc/glossary.rst b/Doc/glossary.rst index c0ca0be304e..a4066d42927 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1025,6 +1025,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 00000000000..262e73f2eaa --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43feba..12650861c0f 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -25,6 +25,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index a901403621b..89b6fe540f5 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -33,6 +33,8 @@ The :mod:`!compression.zstd` module contains: * The :class:`CompressionParameter`, :class:`DecompressionParameter`, and :class:`Strategy` classes for setting advanced (de)compression parameters. +.. include:: ../includes/optional-module.rst + Exceptions ---------- diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d8dac24c8ab..9c0b246c095 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,8 @@ data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + .. _ctypes-ctypes-tutorial: diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index e60197ddd89..057d338edda 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -23,6 +23,8 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: Whenever the documentation mentions a *character* it can be specified diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index fa102c4a080..165b9a9f823 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -30,6 +30,8 @@ when creating a virtual environment) or after explicitly uninstalling needed to bootstrap ``pip`` are included as internal parts of the package. +.. include:: ../includes/optional-module.rst + .. seealso:: :ref:`installing-index` diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 4bdcec66088..cb36be42a83 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -11,6 +11,8 @@ This module provides a simple interface to compress and decompress files just like the GNU programs :program:`gzip` and :program:`gunzip` would. +.. include:: ../includes/optional-module.rst + The data compression is provided by the :mod:`zlib` module. The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 10ec7f0a6f1..52e3726a0f5 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -37,6 +37,10 @@ IDLE has the following features: * configuration, browsers, and other dialogs +The IDLE application is implemented in the :mod:`idlelib` package. + +.. include:: ../includes/optional-module.rst + Menus ----- diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 69f7cb8d48d..8a4f68f3502 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -23,6 +23,8 @@ module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. +.. include:: ../includes/optional-module.rst + .. exception:: LZMAError diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index f649fce5efc..75db832c546 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -26,6 +26,8 @@ Readline library in general. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: The underlying Readline library API may be implemented by diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9d56e81dee1..3b1a9c2f6ee 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -31,7 +31,9 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.15.2 or newer. +requires the third-party `SQLite `_ library. + +.. include:: ../includes/optional-module.rst This document includes four main sections: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index e0d85c852fa..fa0a5234720 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -18,8 +18,9 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. +library. + +.. include:: ../includes/optional-module.rst .. note:: diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index c4614bf28a4..5ff8502bbe2 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -21,6 +21,14 @@ Some facts and figures: * reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and :mod:`lzma` compressed archives if the respective modules are available. + .. + The following paragraph should be similar to ../includes/optional-module.rst + + If any of these :term:`optional modules ` are missing from + your copy of CPython, look for documentation from your distributor (that is, + whoever provided Python to you). + If you are the distributor, see :ref:`optional-module-requirements`. + * read/write support for the POSIX.1-1988 (ustar) format. * read/write support for the GNU tar format including *longname* and *longlink* diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 22e08c45d01..81177533be8 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -36,6 +36,8 @@ details that are unchanged. Most documentation you will find online still uses the old API and can be woefully outdated. +.. include:: ../includes/optional-module.rst + .. seealso:: * `TkDocs `_ diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index b687231bd48..58b99e0d441 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -29,6 +29,8 @@ introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon in 1967. +.. include:: ../includes/optional-module.rst + Get started =========== diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index f6ec33640b6..5a8bbc8c1ae 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -23,6 +23,16 @@ decryption of encrypted files in ZIP archives, but it currently cannot create an encrypted file. Decryption is extremely slow as it is implemented in native Python rather than C. +.. + The following paragraph should be similar to ../includes/optional-module.rst + +Handling compressed archives requires :term:`optional modules ` +such as :mod:`zlib`, :mod:`bz2`, :mod:`lzma`, and :mod:`compression.zstd`. +If any of them are missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + The module defines the following items: .. exception:: BadZipFile diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index b961f7113d3..ce0a22b9456 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -8,9 +8,9 @@ -------------- For applications that require data compression, the functions in this module -allow compression and decompression, using the zlib library. The zlib library -has its own home page at https://www.zlib.net. zlib 1.2.2.1 is the minium -supported version. +allow compression and decompression, using the `zlib library `_. + +.. include:: ../includes/optional-module.rst zlib's functions have many options and often need to be used in a particular order. This documentation doesn't attempt to cover all of the permutations; diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 1f773a3a547..cdadbe51417 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -4,10 +4,13 @@ Configure Python .. highlight:: sh + +.. _build-requirements: + Build Requirements ================== -Features and minimum versions required to build CPython: +To build CPython, you will need: * A `C11 `_ compiler. `Optional C11 features @@ -22,85 +25,138 @@ Features and minimum versions required to build CPython: * Support for threads. -To build optional modules: - -* `libbz2 `_ for the :mod:`bz2` module. - -* `libb2 `_ (:ref:`BLAKE2 `) - for the :mod:`hashlib` module. - -* `libffi `_ 3.3.0 is the recommended - minimum version for the :mod:`ctypes` module. - -* ``liblzma`` for the :mod:`lzma` module. - -* `libmpdec `_ 2.5.0 - for the :mod:`decimal` module. - -* ``libncurses`` or ``libncursesw`` for the :mod:`curses` module. - -* ``libpanel`` or ``libpanelw`` for the :mod:`curses.panel` module. - -* `libreadline `_ or - `libedit `_ - for the :mod:`readline` module. - -* `libuuid `_ for the :mod:`uuid` module. - -* `OpenSSL `_ 1.1.1 is the minimum version and - OpenSSL 3.0.18 is the recommended minimum version for the - :mod:`ssl` and :mod:`hashlib` extension modules. - -* `SQLite `_ 3.15.2 for the :mod:`sqlite3` extension module. - -* `Tcl/Tk `_ 8.5.12 for the :mod:`tkinter` module. - -* `zlib `_ 1.2.2.1 is the minimum version for the - :mod:`zlib` module. - -* `zstd `_ 1.4.5 is the minimum version for - the :mod:`compression.zstd` module. - -For a full list of dependencies required to build all modules and how to install -them, see the -`devguide `_. - -* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the - :file:`configure` script. - -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. - .. versionchanged:: 3.5 On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. .. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. + Selected C99 features, like ```` and ``static inline`` functions, + are now required. .. versionchanged:: 3.7 - Thread support and OpenSSL 1.0.2 are now required. - -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. + Thread support is now required. .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. - -.. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. - -.. versionchanged:: 3.14 - Autoconf 2.72 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". +.. _optional-module-requirements: + +Requirements for optional modules +--------------------------------- + +Some :term:`optional modules ` of the standard library +require third-party libraries installed for development +(for example, header files must be available). + +Missing requirements are reported in the ``configure`` output. +Modules that are missing due to missing dependencies are listed near the end +of the ``make`` output, +sometimes using an internal name, for example, ``_ctypes`` for :mod:`ctypes` +module. + +If you distribute a CPython interpreter without optional modules, +it's best practice to advise users, who generally expect that +standard library modules are available. + +Dependencies to build optional modules are: + +.. list-table:: + :header-rows: 1 + :align: left + + * - Dependency + - Minimum version + - Python module + * - `libbz2 `_ + - + - :mod:`bz2` + * - `libffi `_ + - 3.3.0 recommended + - :mod:`ctypes` + * - `liblzma `_ + - + - :mod:`lzma` + * - `libmpdec `_ + - 2.5.0 + - :mod:`decimal` [1]_ + * - `libreadline `_ or + `libedit `_ [2]_ + - + - :mod:`readline` + * - `libuuid `_ + - + - ``_uuid`` [3]_ + * - `ncurses `_ [4]_ + - + - :mod:`curses` + * - `OpenSSL `_ + - | 3.0.18 recommended + | (1.1.1 minimum) + - :mod:`ssl`, :mod:`hashlib` [5]_ + * - `SQLite `_ + - 3.15.2 + - :mod:`sqlite3` + * - `Tcl/Tk `_ + - 8.5.12 + - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` + * - `zlib `_ + - 1.2.2.1 + - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` + * - `zstd `_ + - 1.4.5 + - :mod:`compression.zstd` + +.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use + a pure-Python implementation. + See :option:`--with-system-libmpdec` for details. +.. [2] See :option:`--with-readline` for choosing the backend for the + :mod:`readline` module. +.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. + See the module documentation for details. +.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` + library. + The :mod:`curses.panel` module additionally requires the ``libpanel`` or + ``libpanelw`` library. +.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use + bundled implementations of several hash functions. + See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. + +Note that the table does not include all optional modules; in particular, +platform-specific modules like :mod:`winreg` are not listed here. + +.. seealso:: + + * The `devguide `_ + includes a full list of dependencies required to build all modules and + instructions on how to install them on common platforms. + * :option:`--with-system-expat` allows building with an external + `libexpat `_ library. + * :ref:`configure-options-for-dependencies` + +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.5 + Tcl/Tk version 8.4 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.7 + OpenSSL 1.0.2 is now required for :mod:`hashlib` and :mod:`ssl`. + +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required for :mod:`hashlib` and :mod:`ssl`. + SQLite 3.7.15 is now required for :mod:`sqlite3`. + +.. versionchanged:: 3.11 + Tcl/Tk version 8.5.12 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.13 + SQLite 3.15.2 is now required for :mod:`sqlite3`. + + Generated files =============== @@ -127,8 +183,19 @@ The container is optional, the following command can be run locally:: autoreconf -ivf -Werror -The generated files can change depending on the exact ``autoconf-archive``, -``aclocal`` and ``pkg-config`` versions. +The generated files can change depending on the exact versions of the +tools used. +The container that CPython uses has +`Autoconf `_ 2.72, +``aclocal`` from `Automake `_ 1.16.5, +and `pkg-config `_ 1.8.1. + +.. versionchanged:: 3.13 + Autoconf 2.71 and aclocal 1.16.5 and are now used to regenerate + :file:`configure`. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now used to regenerate :file:`configure`. .. _configure-options: @@ -409,6 +476,8 @@ Linker options Name for machine-dependent library files. +.. _configure-options-for-dependencies: + Options for third-party dependencies ------------------------------------ @@ -431,12 +500,6 @@ Options for third-party dependencies C compiler and linker flags for ``gdbm``. -.. option:: LIBB2_CFLAGS -.. option:: LIBB2_LIBS - - C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), - used by :mod:`hashlib` module, overriding ``pkg-config``. - .. option:: LIBEDIT_CFLAGS .. option:: LIBEDIT_LIBS @@ -902,6 +965,13 @@ Libraries options .. versionchanged:: 3.13 Default to using the installed ``mpdecimal`` library. + .. versionchanged:: 3.15 + + A bundled copy of the library will no longer be selected + implicitly if an installed ``mpdecimal`` library is not found. + In Python 3.15 only, it can still be selected explicitly using + ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. + .. deprecated-removed:: 3.13 3.16 A copy of the ``mpdecimal`` library sources will no longer be distributed with Python 3.16. diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 9ec4e341932..a9950ef7525 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -84,11 +84,17 @@ On FreeBSD and OpenBSD Building Python =============== +.. seealso:: + + If you want to contribute to CPython, refer to the + `devguide `_, + which includes build instructions and other tips on setting up environment. + If you want to compile CPython yourself, first thing you should do is get the `source `_. You can download either the -latest release's source or just grab a fresh `clone -`_. (If you want -to contribute patches, you will need a clone.) +latest release's source or grab a fresh `clone +`_. +You will also need to install the :ref:`build requirements `. The build process consists of the usual commands:: From 1697cb5710f526d38816bb00ca3dcd4434e5e773 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Nov 2025 07:29:04 -0500 Subject: [PATCH 050/417] gh-141004: Document built-in iterator types in the C API (GH-141006) Adds documentation for each of the following: - PyEnum_Type - PyFilter_Type - PyMap_Type - PyReversed_Type - PyZip_Type In addition, PyRange_Type and PyRange_Check are also documented. --- Doc/c-api/iterator.rst | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 4b94970036f..7eaf72ec55f 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -52,6 +52,45 @@ sentinel value is returned. *sentinel*, the iteration will be terminated. +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + Other Iterator Objects ^^^^^^^^^^^^^^^^^^^^^^ From 54110e20e0ed0584e159c42d9f57516c1a3b997a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 6 Nov 2025 13:48:42 +0000 Subject: [PATCH 051/417] gh-141004: Document `Py_hexdigits` (GH-141059) Co-authored-by: Victor Stinner --- Doc/c-api/codec.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 08a99245ad6..35ee048bd5f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 From bcc524f82d8548707046ce90f5bc56f60018767a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Nov 2025 09:01:48 -0500 Subject: [PATCH 052/417] gh-141004: Document `PyLong_FromPid` and `PyLong_AsPid` (GH-141028) Co-authored-by: Victor Stinner --- Doc/c-api/long.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index fcb20f7c93c..ed34efe716d 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -161,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -575,6 +586,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. From 2e5e6fd380eb747bffeac151ca6f609779ef36f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Nov 2025 16:10:39 +0100 Subject: [PATCH 053/417] gh-134745: Use "pymutex" for sys.thread_info on Windows (#141140) --- Python/thread.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Python/thread.c b/Python/thread.c index 18c4af7f634..0365f977d82 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -334,14 +334,12 @@ PyThread_GetInfo(void) #ifdef HAVE_PTHREAD_STUBS value = Py_NewRef(Py_None); -#elif defined(_POSIX_THREADS) +#else value = PyUnicode_FromString("pymutex"); if (value == NULL) { Py_DECREF(threadinfo); return NULL; } -#else - value = Py_NewRef(Py_None); #endif PyStructSequence_SET_ITEM(threadinfo, pos++, value); From 13f09a60f4d6fca019bff9dec05dabe5ad390d21 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 6 Nov 2025 10:16:56 -0500 Subject: [PATCH 054/417] =?UTF-8?q?gh-141150:=20Don't=20rely=20on=20implic?= =?UTF-8?q?it=20conversion=20from=20void=20*=20to=20pointer=20in=20=5FPyMo?= =?UTF-8?q?dule=E2=80=A6=20(#141147)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't rely on implicit conversion from void * to pointer in _PyModule_GetToken --- Include/internal/pycore_moduleobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index c34e42e826e..6eef6eaa5df 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -53,7 +53,7 @@ static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { PyModuleObject *mod = _PyModule_CAST(arg); - return mod->md_token; + return (PyModuleDef *)mod->md_token; } static inline void* _PyModule_GetState(PyObject* mod) { From bea0d3d12bcd122d8498b92cdd6c724822fd6505 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 7 Nov 2025 00:33:30 +0900 Subject: [PATCH 055/417] gh-140826: Update winreg's docstring (GH-141050) --- PC/winreg.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/PC/winreg.c b/PC/winreg.c index 3cc6123fc3a..c1be920fc1d 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -53,6 +53,7 @@ PyDoc_STRVAR(module_doc, "DeleteKey() - Deletes the specified key.\n" "DeleteKeyEx() - Deletes the specified key.\n" "DeleteValue() - Removes a named value from the specified registry key.\n" +"DeleteTree() - Deletes the specified key and all its subkeys and values recursively.\n" "EnumKey() - Enumerates subkeys of the specified open registry key.\n" "EnumValue() - Enumerates values of the specified open registry key.\n" "ExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n" @@ -107,7 +108,9 @@ PyDoc_STRVAR(PyHKEY_doc, "Operations:\n" "__bool__ - Handles with an open object return true, otherwise false.\n" "__int__ - Converting a handle to an integer returns the Win32 handle.\n" -"rich comparison - Handle objects are compared using the handle value."); +"__enter__, __exit__ - Context manager support for 'with' statement,\n" +"automatically closes handle.\n" +"__eq__, __ne__ - Equality comparison based on Windows handle value."); From 0b260305d302eace7d59931ca582a1953d894018 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Nov 2025 11:37:52 -0500 Subject: [PATCH 056/417] gh-141004: Document `Py_GetRecursionLimit` and `Py_SetRecursionLimit` (GH-141151) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 3ff4631a8e5..9c75f66f683 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -979,6 +979,27 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: From e95e783dff443b68e8179fdb57737025bf02ba76 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 6 Nov 2025 18:48:58 +0200 Subject: [PATCH 057/417] Remove duplicated tests in test_base64 (gh-125346) (GH-141153) --- Lib/test/test_base64.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 65977ca8c9f..ac3f0940545 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -394,10 +394,6 @@ def test_b32decode_casefold(self): self.assertRaises(binascii.Error, base64.b32decode, b'me======') self.assertRaises(binascii.Error, base64.b32decode, 'me======') - # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - def test_b32decode_map01(self): # Mapping zero and one eq = self.assertEqual From 9bf5100037f661f3a369d3ee539bec06f063b650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20Gir=C3=A3o=20Serr=C3=A3o?= <5621605+rodrigogiraoserrao@users.noreply.github.com> Date: Thu, 6 Nov 2025 17:11:50 +0000 Subject: [PATCH 058/417] Minor documentation improvements (#140626) --- Doc/library/concurrent.interpreters.rst | 6 +++--- Doc/library/heapq.rst | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 41ea6af3b22..55036090e8d 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -29,12 +29,12 @@ Actual concurrency is available separately through .. seealso:: :class:`~concurrent.futures.InterpreterPoolExecutor` - combines threads with interpreters in a familiar interface. + Combines threads with interpreters in a familiar interface. - .. XXX Add references to the upcoming HOWTO docs in the seealso block. + .. XXX Add references to the upcoming HOWTO docs in the seealso block. :ref:`isolating-extensions-howto` - how to update an extension module to support multiple interpreters + How to update an extension module to support multiple interpreters. :pep:`554` diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 95ef72469b1..5049262306a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -58,6 +58,11 @@ functions, respectively. The following functions are provided for min-heaps: +.. function:: heapify(x) + + Transform list *x* into a min-heap, in-place, in linear time. + + .. function:: heappush(heap, item) Push the value *item* onto the *heap*, maintaining the min-heap invariant. @@ -77,11 +82,6 @@ The following functions are provided for min-heaps: followed by a separate call to :func:`heappop`. -.. function:: heapify(x) - - Transform list *x* into a min-heap, in-place, in linear time. - - .. function:: heapreplace(heap, item) Pop and return the smallest item from the *heap*, and also push the new *item*. From 42d014086098d3d70cacb4d8993f04cace120c12 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 6 Nov 2025 11:58:01 -0800 Subject: [PATCH 059/417] GH-136895: Fixes for pulling LLVM as a release artifact (#141002) --- PCbuild/get_external.py | 37 ++++++++++++++++++++----------------- Tools/jit/_llvm.py | 5 +++++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 07970624e86..edf14ce578b 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,8 +3,8 @@ import argparse import os import pathlib -import shutil import sys +import tarfile import time import urllib.error import urllib.request @@ -56,7 +56,8 @@ def fetch_release(tag, tarball_dir, *, org='python', verbose=False): def extract_tarball(externals_dir, tarball_path, tag): output_path = externals_dir / tag - shutil.unpack_archive(os.fspath(tarball_path), os.fspath(output_path)) + with tarfile.open(tarball_path) as tf: + tf.extractall(os.fspath(externals_dir)) return output_path @@ -115,21 +116,23 @@ def main(): verbose=args.verbose, ) extracted = extract_zip(args.externals_dir, zip_path) - for wait in [1, 2, 3, 5, 8, 0]: - try: - extracted.replace(final_name) - break - except PermissionError as ex: - retry = f" Retrying in {wait}s..." if wait else "" - print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) - time.sleep(wait) - else: - print( - f"ERROR: Failed to extract {final_name}.", - "You may need to restart your build", - file=sys.stderr, - ) - sys.exit(1) + + if extracted != final_name: + for wait in [1, 2, 3, 5, 8, 0]: + try: + extracted.replace(final_name) + break + except PermissionError as ex: + retry = f" Retrying in {wait}s..." if wait else "" + print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) + time.sleep(wait) + else: + print( + f"ERROR: Failed to rename {extracted} to {final_name}.", + "You may need to restart your build", + file=sys.stderr, + ) + sys.exit(1) if __name__ == '__main__': diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 54c2bf86a36..f1b0ad3f5db 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -83,6 +83,11 @@ async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) + # On Windows, executables need .exe extension + if os.name == "nt" and not path.endswith(".exe"): + path_with_exe = path + ".exe" + if os.path.exists(path_with_exe): + path = path_with_exe if await _check_tool_version(path, llvm_version, echo=echo): return path # Homebrew-installed executables: From c77441ef1d1f3182280bd14d11516d54f38fe90b Mon Sep 17 00:00:00 2001 From: yihong Date: Fri, 7 Nov 2025 08:02:47 +0800 Subject: [PATCH 060/417] gh-141125: delete unused import textwrap in interpreter.py (#141126) --- Lib/concurrent/futures/interpreter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 53c6e757ded..85c1da2c722 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -2,7 +2,6 @@ from concurrent import interpreters import sys -import textwrap from . import thread as _thread import traceback From 9a199006733dae999f96c0f596c2035f4b9847b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 7 Nov 2025 10:54:02 +0100 Subject: [PATCH 061/417] gh-140734: fix off-by-one error when comparing to `_SUN_PATH_MAX` (#140903) The limit includes a NULL terminator. --- Lib/multiprocessing/util.py | 20 +++++++++++-------- ...-11-02-09-37-22.gh-issue-140734.f8gST9.rst | 2 ++ 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index a1a537dd48d..549fb07c275 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -126,12 +126,14 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # -# Maximum length of a socket file path is usually between 92 and 108 [1], -# but Linux is known to use a size of 108 [2]. BSD-based systems usually -# use a size of 104 or 108 and Windows does not create AF_UNIX sockets. +# Maximum length of a NULL-terminated [1] socket file path is usually +# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. +# BSD-based systems usually use a size of 104 or 108 and Windows does +# not create AF_UNIX sockets. # -# [1]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html -# [2]: https://man7.org/linux/man-pages/man7/unix.7.html. +# [1]: https://github.com/python/cpython/issues/140734 +# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [3]: https://man7.org/linux/man-pages/man7/unix.7.html if sys.platform == 'linux': _SUN_PATH_MAX = 108 @@ -171,11 +173,13 @@ def _get_base_temp_dir(tempfile): # generated by tempfile._RandomNameSequence, which, by design, # is 8 characters long. # - # Thus, the length of socket filename will be: + # Thus, the socket file path length (without NULL terminator) will be: # # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') sun_path_len = len(base_tempdir) + 14 + 14 - if sun_path_len <= _SUN_PATH_MAX: + # Strict inequality to account for the NULL terminator. + # See https://github.com/python/cpython/issues/140734. + if sun_path_len < _SUN_PATH_MAX: return base_tempdir # Fallback to the default system-wide temporary directory. # This ignores user-defined environment variables. @@ -201,7 +205,7 @@ def _get_base_temp_dir(tempfile): return base_tempdir warn("Ignoring user-defined temporary directory: %s", base_tempdir) # at most max(map(len, dirlist)) + 14 + 14 = 36 characters - assert len(base_system_tempdir) + 14 + 14 <= _SUN_PATH_MAX + assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX return base_system_tempdir def get_temp_dir(): diff --git a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst new file mode 100644 index 00000000000..46582f7fcf4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst @@ -0,0 +1,2 @@ +:mod:`multiprocessing`: fix off-by-one error when checking the length +of a temporary socket file path. Patch by Bénédikt Tran. From a7bf27f7f521384a8964718bdb58a5cb113bb3ec Mon Sep 17 00:00:00 2001 From: Benel Tayar <86257734+beneltayar@users.noreply.github.com> Date: Fri, 7 Nov 2025 12:47:25 +0200 Subject: [PATCH 062/417] gh-141141: Make base64.b85decode() thread safe (GH-141149) --- Lib/base64.py | 7 +++++-- .../Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst diff --git a/Lib/base64.py b/Lib/base64.py index 5d78cc09f40..cfc57626c40 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -462,9 +462,12 @@ def b85decode(b): # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: - _b85dec = [None] * 256 + # we don't assign to _b85dec directly to avoid issues when + # multiple threads call this function simultaneously + b85dec_tmp = [None] * 256 for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + b85dec_tmp[c] = i + _b85dec = b85dec_tmp b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 diff --git a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst new file mode 100644 index 00000000000..f59ccfb33e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst @@ -0,0 +1 @@ +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by Benel Tayar. From ffd64737d00277eea1c4721d278a0951168d07ca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 7 Nov 2025 14:17:47 +0100 Subject: [PATCH 063/417] Clarify argument/result ownership/validity for PyModule_* functions (GH-141159) Co-authored-by: Peter Bierma --- Doc/c-api/module.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index ed2a7663375..1994a3c7d01 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -13,7 +13,7 @@ Module Objects .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +71,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,6 +93,10 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. + .. c:function:: void* PyModule_GetState(PyObject *module) Return the "state" of the module, that is, a pointer to the block of memory @@ -126,6 +133,9 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. @@ -671,6 +681,9 @@ or code that creates modules dynamically. :c:type:`PyMethodDef` arrays; in that case they should call this function directly. + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + .. versionadded:: 3.5 .. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) From 920286d6b296f9971fc79e14ec22966f8f7a7b90 Mon Sep 17 00:00:00 2001 From: "W. H. Wang" Date: Fri, 7 Nov 2025 21:34:49 +0800 Subject: [PATCH 064/417] Update NaNs handling description in `c-api/float.rst` (#141179) Clarified the behavior of NaNs on IEEE platforms regarding signaling and quiet NaNs. --- Doc/c-api/float.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 1085c32a537..edee498a0b8 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -113,8 +113,8 @@ NaNs (if such things exist on the platform) isn't handled correctly, and attempting to unpack a bytes string containing an IEEE INF or NaN will raise an exception. -Note that NaNs type may not be preserved on IEEE platforms (silent NaN become -quiet), for example on x86 systems in 32-bit mode. +Note that NaNs type may not be preserved on IEEE platforms (signaling NaN become +quiet NaN), for example on x86 systems in 32-bit mode. On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less From 7af9b5354dd7633df422b9f720633989b3090199 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 7 Nov 2025 09:09:38 -0500 Subject: [PATCH 065/417] gh-141004: Document `PyCapsule_Type` (GH-141079) --- Doc/c-api/capsule.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 6da020efc7f..03a848d68ed 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -22,6 +22,12 @@ Refer to :ref:`using-capsules` for more information on using these objects. loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: From 9420795b47ac88f31315a8d1041e2c66c2cd9a8b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 7 Nov 2025 11:19:14 -0500 Subject: [PATCH 066/417] gh-141004: Document `PyErr_WarnExplicitFormat` (GH-141187) --- Doc/c-api/exceptions.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 9c75f66f683..b6f9399337e 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -394,6 +394,15 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is From 3989e12d39bfe2587e5ba80873c37e0c2d449088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 7 Nov 2025 19:25:32 +0100 Subject: [PATCH 067/417] gh-141004: Document `Py_HASH_*` macros (#141205) --- Doc/c-api/hash.rst | 56 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index b5fe93573a1..e74f5b4d18e 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,42 +11,94 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 -.. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. .. versionadded:: 3.13 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + .. versionadded:: 3.4 + + +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + This corresponds to the :data:`sys.hash_info.modulus` constant. + + .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. From d13ee0ae186f4704f3b6016dd52f7727b81f9194 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 7 Nov 2025 13:46:47 -0500 Subject: [PATCH 068/417] gh-141004: Document `PyTraceBack*` APIs (GH-141192) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index b6f9399337e..f525ee7a046 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1237,3 +1237,37 @@ Warning types .. versionadded:: 3.10 :c:data:`PyExc_EncodingWarning`. + + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. From cd0b3e5d12ed57d41878a89aac581375d021e25a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 8 Nov 2025 06:28:01 +0000 Subject: [PATCH 069/417] gh-140849: Update bundled liblzma to 5.8.1 on Windows (#141022) --- .../2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst | 1 + Misc/externals.spdx.json | 8 ++++---- PCbuild/get_externals.bat | 2 +- PCbuild/liblzma.vcxproj | 7 +++---- PCbuild/liblzma.vcxproj.filters | 11 ++++------- PCbuild/python.props | 2 +- 6 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst diff --git a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst new file mode 100644 index 00000000000..6f25b867566 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst @@ -0,0 +1 @@ +Update bundled liblzma to version 5.8.1. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 59aceedb94d..dba01de8352 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -154,21 +154,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + "checksumValue": "1bfaba0ccacc6681d3ba85335cc7f49c24cf6f9d16f848cbd153b896d8a7d631" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.8.1.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tukaani:xz:5.2.5:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tukaani:xz:5.8.1.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "xz", "primaryPackagePurpose": "SOURCE", - "versionInfo": "5.2.5" + "versionInfo": "5.8.1.1" }, { "SPDXID": "SPDXRef-PACKAGE-zlib-ng", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 319024e0f50..115203cecc8 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -59,7 +59,7 @@ set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 -set libraries=%libraries% xz-5.2.5 +set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 diff --git a/PCbuild/liblzma.vcxproj b/PCbuild/liblzma.vcxproj index 97938692328..75d4e162346 100644 --- a/PCbuild/liblzma.vcxproj +++ b/PCbuild/liblzma.vcxproj @@ -92,7 +92,7 @@ WIN32;HAVE_CONFIG_H;_LIB;%(PreprocessorDefinitions) - $(lzmaDir)windows/vs2019;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories) + $(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories) 4244;4267;4996;%(DisableSpecificWarnings) %(AdditionalOptions) -Wno-deprecated-declarations @@ -102,9 +102,7 @@ - - @@ -163,6 +161,7 @@ + @@ -239,7 +238,7 @@ - + diff --git a/PCbuild/liblzma.vcxproj.filters b/PCbuild/liblzma.vcxproj.filters index ebe2a7d5fa9..42feca5a341 100644 --- a/PCbuild/liblzma.vcxproj.filters +++ b/PCbuild/liblzma.vcxproj.filters @@ -18,6 +18,9 @@ Source Files + + Source Files + Source Files @@ -54,15 +57,9 @@ Source Files - - Source Files - Source Files - - Source Files - Source Files @@ -428,7 +425,7 @@ Header Files - + Header Files diff --git a/PCbuild/python.props b/PCbuild/python.props index cc157252655..7840e2a1cfc 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -77,7 +77,7 @@ $(ExternalsDir)sqlite-3.50.4.0\ $(ExternalsDir)bzip2-1.0.8\ - $(ExternalsDir)xz-5.2.5\ + $(ExternalsDir)xz-5.8.1.1\ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include From c1785129c3b1b020e1523ff4f0d4314b9854d0e3 Mon Sep 17 00:00:00 2001 From: Brandon Hubacher Date: Sat, 8 Nov 2025 03:08:19 -0600 Subject: [PATCH 070/417] fix typos in contextvars asyncio support example docs (#141219) Co-authored-by: Kumar Aditya --- Doc/library/contextvars.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 57580ce026e..b218468a084 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -313,7 +313,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is now possible to get + # In any code that we call, it is now possible to get the # client's address by calling 'client_addr_var.get()'. while True: From 8cec3d3a9d827aadc7008ab4312121fcf28329c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Nov 2025 10:10:27 +0100 Subject: [PATCH 071/417] gh-141004: fix `versionadded` typo for `Py_HASH_SIPHASH13` (#141223) --- Doc/c-api/hash.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index e74f5b4d18e..ecd604c81bc 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -40,7 +40,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.4 Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. - .. versionadded:: 3.13 + .. versionadded:: 3.11 Add :c:macro:`!Py_HASH_SIPHASH13`. From 7e90bac3cc6fd68fe6696ab4bce1262751de7531 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Nov 2025 12:07:27 +0200 Subject: [PATCH 072/417] gh-140793: Improve documentatation and tests for the ensure_ascii option in the json module (GH-140906) * Document that ensure_ascii=True forces escaping not only non-ASCII, but also non-printable characters (the only affected ASCII character is U+007F). * Ensure that the help output for the json module does not exceed 80 columns (except one long line in an example and generated lines). * Add more tests. --- Doc/library/json.rst | 14 +++-- Lib/json/__init__.py | 52 +++++++++++-------- Lib/json/decoder.py | 8 +-- Lib/json/encoder.py | 20 +++---- .../test_json/test_encode_basestring_ascii.py | 3 +- Lib/test/test_json/test_unicode.py | 35 +++++++++++++ 6 files changed, 89 insertions(+), 43 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 12a5a96a3c5..8b4217c210d 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -183,8 +183,10 @@ Basic Usage :param bool ensure_ascii: If ``True`` (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. - If ``False``, these characters will be outputted as-is. + have all incoming non-ASCII and non-printable characters escaped. + If ``False``, all characters will be outputted as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. :param bool check_circular: If ``False``, the circular reference check for container types is skipped @@ -495,8 +497,10 @@ Encoders and Decoders :class:`bool` or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + have all incoming non-ASCII and non-printable characters escaped. + If *ensure_ascii* is false, all characters will be output as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -636,7 +640,7 @@ UTF-32, with UTF-8 being the recommended default for maximum interoperability. As permitted, though not required, by the RFC, this module's serializer sets *ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. +strings only contain printable ASCII characters. Other than the *ensure_ascii* parameter, this module is defined strictly in terms of conversion between Python objects and diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index c8fdd0d99a0..89396b25a2c 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -127,8 +127,9 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can - contain non-ASCII characters if they appear in strings contained in - ``obj``. Otherwise, all such characters are escaped in JSON strings. + contain non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in JSON + strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -144,10 +145,11 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -188,9 +190,10 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is false, then the return value can contain non-ASCII - characters if they appear in strings contained in ``obj``. Otherwise, all - such characters are escaped in JSON strings. + If ``ensure_ascii`` is false, then the return value can contain + non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in + JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -206,10 +209,11 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -280,11 +284,12 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -305,11 +310,12 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index ff4bfcdcc40..92ad6352557 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -297,10 +297,10 @@ def __init__(self, *, object_hook=None, parse_float=None, place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). - ``object_pairs_hook``, if specified will be called with the result of - every JSON object decoded with an ordered list of pairs. The return - value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. + ``object_pairs_hook``, if specified will be called with the result + of every JSON object decoded with an ordered list of pairs. The + return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index bc446e0f377..5cf6d64f3ea 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -111,9 +111,10 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, encoding of keys that are not str, int, float, bool or None. If skipkeys is True, such items are simply skipped. - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming non-ASCII characters escaped. If - ensure_ascii is false, the output can contain non-ASCII characters. + If ensure_ascii is true, the output is guaranteed to be str objects + with all incoming non-ASCII and non-printable characters escaped. + If ensure_ascii is false, the output can contain non-ASCII and + non-printable characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -134,14 +135,15 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, indent level. An indent level of 0 will only insert newlines. None is the most compact representation. - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. + If specified, separators should be an (item_separator, + key_separator) tuple. The default is (', ', ': ') if *indent* is + ``None`` and (',', ': ') otherwise. To get the most compact JSON + representation, you should specify (',', ':') to eliminate + whitespace. If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. + that can't otherwise be serialized. It should return a JSON + encodable version of the object or raise a ``TypeError``. """ diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 6a39b72a09d..c90d3e968e5 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -8,13 +8,12 @@ ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ('controls', '"controls"'), ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\x00\x1f\x7f', '"\\u0000\\u001f\\u007f"'), ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (' s p a c e d ', '" s p a c e d "'), ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.?", '"`1~!@#$%^&*()_+-={\':[,]}|;.?"'), - ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBasestringAscii: diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 68629cceeb9..1aa9546dc46 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -32,6 +32,29 @@ def test_encoding7(self): j = self.dumps(u + "\n", ensure_ascii=False) self.assertEqual(j, f'"{u}\\n"') + def test_ascii_non_printable_encode(self): + u = '\b\t\n\f\r\x00\x1f\x7f' + self.assertEqual(self.dumps(u), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') + self.assertEqual(self.dumps(u, ensure_ascii=False), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') + + def test_ascii_non_printable_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), + '\b\t\n\f\r') + s = ''.join(map(chr, range(32))) + for c in s: + self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + self.assertEqual(self.loads(f'"{s}"', strict=False), s) + self.assertEqual(self.loads('"\x7f"'), '\x7f') + + def test_escaped_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') + self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') + for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) + def test_big_unicode_encode(self): u = '\U0001d120' self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') @@ -48,6 +71,18 @@ def test_unicode_decode(self): s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) + def test_single_surrogate_encode(self): + self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') + self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') + self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') + self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') + + def test_single_surrogate_decode(self): + self.assertEqual(self.loads('"\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\udc0d"'), '\udc0d') + self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') + def test_unicode_preservation(self): self.assertEqual(type(self.loads('""')), str) self.assertEqual(type(self.loads('"a"')), str) From 610377056bad696915d70590429e68002bee9006 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 8 Nov 2025 12:17:59 +0200 Subject: [PATCH 073/417] gh-140615: Update docstrings in the fcntl module (GH-140619) * Refer to bytes objects or bytes-like objects instead of strings. * Remove backticks -- they do not have effect on formatting. * Re-wrap lines to ensure the pydoc output fits in 80 coluimns. * Remove references to the 1024 bytes limit. --- Modules/clinic/fcntlmodule.c.h | 90 ++++++++++++++++--------------- Modules/fcntlmodule.c | 98 +++++++++++++++++----------------- 2 files changed, 97 insertions(+), 91 deletions(-) diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 2b61d9f8708..718f80bfe73 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -8,17 +8,22 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" "\n" -"Perform the operation `cmd` on file descriptor fd.\n" +"Perform the operation cmd on file descriptor fd.\n" "\n" -"The values used for `cmd` are operating system dependent, and are available\n" -"as constants in the fcntl module, using the same names as used in\n" -"the relevant C header files. The argument arg is optional, and\n" -"defaults to 0; it may be an int or a string. If arg is given as a string,\n" -"the return value of fcntl is a string of that length, containing the\n" -"resulting value put in the arg buffer by the operating system. The length\n" -"of the arg string is not allowed to exceed 1024 bytes. If the arg given\n" -"is an integer or if none is specified, the result value is an integer\n" -"corresponding to the return value of the fcntl call in the C code."); +"The values used for cmd are operating system dependent, and are\n" +"available as constants in the fcntl module, using the same names as used\n" +"in the relevant C header files. The argument arg is optional, and\n" +"defaults to 0; it may be an integer, a bytes-like object or a string.\n" +"If arg is given as a string, it will be encoded to binary using the\n" +"UTF-8 encoding.\n" +"\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the fcntl() call in\n" +"the C code.\n" +"\n" +"If arg is given as a bytes-like object, the return value of fcntl() is a\n" +"bytes object of that length, containing the resulting value put in the\n" +"arg buffer by the operating system."); #define FCNTL_FCNTL_METHODDEF \ {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, @@ -60,34 +65,33 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "ioctl($module, fd, request, arg=0, mutate_flag=True, /)\n" "--\n" "\n" -"Perform the operation `request` on file descriptor `fd`.\n" +"Perform the operation request on file descriptor fd.\n" "\n" -"The values used for `request` are operating system dependent, and are available\n" -"as constants in the fcntl or termios library modules, using the same names as\n" -"used in the relevant C header files.\n" +"The values used for request are operating system dependent, and are\n" +"available as constants in the fcntl or termios library modules, using\n" +"the same names as used in the relevant C header files.\n" "\n" -"The argument `arg` is optional, and defaults to 0; it may be an int or a\n" -"buffer containing character data (most likely a string or an array).\n" +"The argument arg is optional, and defaults to 0; it may be an integer, a\n" +"bytes-like object or a string. If arg is given as a string, it will be\n" +"encoded to binary using the UTF-8 encoding.\n" "\n" -"If the argument is a mutable buffer (such as an array) and if the\n" -"mutate_flag argument (which is only allowed in this case) is true then the\n" -"buffer is (in effect) passed to the operating system and changes made by\n" -"the OS will be reflected in the contents of the buffer after the call has\n" -"returned. The return value is the integer returned by the ioctl system\n" -"call.\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the ioctl() call in\n" +"the C code.\n" "\n" -"If the argument is a mutable buffer and the mutable_flag argument is false,\n" -"the behavior is as if a string had been passed.\n" +"If the argument is a mutable buffer (such as a bytearray) and the\n" +"mutate_flag argument is true (default) then the buffer is (in effect)\n" +"passed to the operating system and changes made by the OS will be\n" +"reflected in the contents of the buffer after the call has returned.\n" +"The return value is the integer returned by the ioctl() system call.\n" "\n" -"If the argument is an immutable buffer (most likely a string) then a copy\n" -"of the buffer is passed to the operating system and the return value is a\n" -"string of the same length containing whatever the operating system put in\n" -"the buffer. The length of the arg buffer in this case is not allowed to\n" -"exceed 1024 bytes.\n" +"If the argument is a mutable buffer and the mutable_flag argument is\n" +"false, the behavior is as if an immutable buffer had been passed.\n" "\n" -"If the arg given is an integer or if none is specified, the result value is\n" -"an integer corresponding to the return value of the ioctl call in the C\n" -"code."); +"If the argument is an immutable buffer then a copy of the buffer is\n" +"passed to the operating system and the return value is a bytes object of\n" +"the same length containing whatever the operating system put in the\n" +"buffer."); #define FCNTL_IOCTL_METHODDEF \ {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, @@ -154,7 +158,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "flock($module, fd, operation, /)\n" "--\n" "\n" -"Perform the lock operation `operation` on file descriptor `fd`.\n" +"Perform the lock operation on file descriptor fd.\n" "\n" "See the Unix manual page for flock(2) for details (On some systems, this\n" "function is emulated using fcntl())."); @@ -195,22 +199,22 @@ PyDoc_STRVAR(fcntl_lockf__doc__, "\n" "A wrapper around the fcntl() locking calls.\n" "\n" -"`fd` is the file descriptor of the file to lock or unlock, and operation is one\n" -"of the following values:\n" +"fd is the file descriptor of the file to lock or unlock, and operation\n" +"is one of the following values:\n" "\n" " LOCK_UN - unlock\n" " LOCK_SH - acquire a shared lock\n" " LOCK_EX - acquire an exclusive lock\n" "\n" "When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\n" -"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the\n" -"lock cannot be acquired, an OSError will be raised and the exception will\n" -"have an errno attribute set to EACCES or EAGAIN (depending on the operating\n" -"system -- for portability, check for either value).\n" +"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and\n" +"the lock cannot be acquired, an OSError will be raised and the exception\n" +"will have an errno attribute set to EACCES or EAGAIN (depending on the\n" +"operating system -- for portability, check for either value).\n" "\n" -"`len` is the number of bytes to lock, with the default meaning to lock to\n" -"EOF. `start` is the byte offset, relative to `whence`, to that the lock\n" -"starts. `whence` is as with fileobj.seek(), specifically:\n" +"len is the number of bytes to lock, with the default meaning to lock to\n" +"EOF. start is the byte offset, relative to whence, to that the lock\n" +"starts. whence is as with fileobj.seek(), specifically:\n" "\n" " 0 - relative to the start of the file (SEEK_SET)\n" " 1 - relative to the current buffer position (SEEK_CUR)\n" @@ -265,4 +269,4 @@ skip_optional: exit: return return_value; } -/*[clinic end generated code: output=9773e44da302dc7c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c782fcf9dd6690e0 input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index df2c9994127..e373bf36881 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -40,22 +40,27 @@ fcntl.fcntl arg: object(c_default='NULL') = 0 / -Perform the operation `cmd` on file descriptor fd. +Perform the operation cmd on file descriptor fd. -The values used for `cmd` are operating system dependent, and are available -as constants in the fcntl module, using the same names as used in -the relevant C header files. The argument arg is optional, and -defaults to 0; it may be an int or a string. If arg is given as a string, -the return value of fcntl is a string of that length, containing the -resulting value put in the arg buffer by the operating system. The length -of the arg string is not allowed to exceed 1024 bytes. If the arg given -is an integer or if none is specified, the result value is an integer -corresponding to the return value of the fcntl call in the C code. +The values used for cmd are operating system dependent, and are +available as constants in the fcntl module, using the same names as used +in the relevant C header files. The argument arg is optional, and +defaults to 0; it may be an integer, a bytes-like object or a string. +If arg is given as a string, it will be encoded to binary using the +UTF-8 encoding. + +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the fcntl() call in +the C code. + +If arg is given as a bytes-like object, the return value of fcntl() is a +bytes object of that length, containing the resulting value put in the +arg buffer by the operating system. [clinic start generated code]*/ static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) -/*[clinic end generated code: output=888fc93b51c295bd input=7955340198e5f334]*/ +/*[clinic end generated code: output=888fc93b51c295bd input=77340720f11665da]*/ { int ret; int async_err = 0; @@ -151,7 +156,6 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) /*[clinic input] -@permit_long_docstring_body fcntl.ioctl fd: fildes @@ -160,40 +164,39 @@ fcntl.ioctl mutate_flag as mutate_arg: bool = True / -Perform the operation `request` on file descriptor `fd`. +Perform the operation request on file descriptor fd. -The values used for `request` are operating system dependent, and are available -as constants in the fcntl or termios library modules, using the same names as -used in the relevant C header files. +The values used for request are operating system dependent, and are +available as constants in the fcntl or termios library modules, using +the same names as used in the relevant C header files. -The argument `arg` is optional, and defaults to 0; it may be an int or a -buffer containing character data (most likely a string or an array). +The argument arg is optional, and defaults to 0; it may be an integer, a +bytes-like object or a string. If arg is given as a string, it will be +encoded to binary using the UTF-8 encoding. -If the argument is a mutable buffer (such as an array) and if the -mutate_flag argument (which is only allowed in this case) is true then the -buffer is (in effect) passed to the operating system and changes made by -the OS will be reflected in the contents of the buffer after the call has -returned. The return value is the integer returned by the ioctl system -call. +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the ioctl() call in +the C code. -If the argument is a mutable buffer and the mutable_flag argument is false, -the behavior is as if a string had been passed. +If the argument is a mutable buffer (such as a bytearray) and the +mutate_flag argument is true (default) then the buffer is (in effect) +passed to the operating system and changes made by the OS will be +reflected in the contents of the buffer after the call has returned. +The return value is the integer returned by the ioctl() system call. -If the argument is an immutable buffer (most likely a string) then a copy -of the buffer is passed to the operating system and the return value is a -string of the same length containing whatever the operating system put in -the buffer. The length of the arg buffer in this case is not allowed to -exceed 1024 bytes. +If the argument is a mutable buffer and the mutable_flag argument is +false, the behavior is as if an immutable buffer had been passed. -If the arg given is an integer or if none is specified, the result value is -an integer corresponding to the return value of the ioctl call in the C -code. +If the argument is an immutable buffer then a copy of the buffer is +passed to the operating system and the return value is a bytes object of +the same length containing whatever the operating system put in the +buffer. [clinic start generated code]*/ static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, int mutate_arg) -/*[clinic end generated code: output=f72baba2454d7a62 input=d7fe504d335449e2]*/ +/*[clinic end generated code: output=f72baba2454d7a62 input=954fe75c208cc492]*/ { /* We use the unsigned non-checked 'I' format for the 'code' parameter because the system expects it to be a 32bit bit field value @@ -340,7 +343,7 @@ fcntl.flock operation as code: int / -Perform the lock operation `operation` on file descriptor `fd`. +Perform the lock operation on file descriptor fd. See the Unix manual page for flock(2) for details (On some systems, this function is emulated using fcntl()). @@ -348,7 +351,7 @@ function is emulated using fcntl()). static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code) -/*[clinic end generated code: output=84059e2b37d2fc64 input=0bfc00f795953452]*/ +/*[clinic end generated code: output=84059e2b37d2fc64 input=ade68943e8599f0a]*/ { int ret; int async_err = 0; @@ -400,7 +403,6 @@ fcntl_flock_impl(PyObject *module, int fd, int code) /*[clinic input] -@permit_long_docstring_body fcntl.lockf fd: fildes @@ -412,22 +414,22 @@ fcntl.lockf A wrapper around the fcntl() locking calls. -`fd` is the file descriptor of the file to lock or unlock, and operation is one -of the following values: +fd is the file descriptor of the file to lock or unlock, and operation +is one of the following values: LOCK_UN - unlock LOCK_SH - acquire a shared lock LOCK_EX - acquire an exclusive lock When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with -LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the -lock cannot be acquired, an OSError will be raised and the exception will -have an errno attribute set to EACCES or EAGAIN (depending on the operating -system -- for portability, check for either value). +LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and +the lock cannot be acquired, an OSError will be raised and the exception +will have an errno attribute set to EACCES or EAGAIN (depending on the +operating system -- for portability, check for either value). -`len` is the number of bytes to lock, with the default meaning to lock to -EOF. `start` is the byte offset, relative to `whence`, to that the lock -starts. `whence` is as with fileobj.seek(), specifically: +len is the number of bytes to lock, with the default meaning to lock to +EOF. start is the byte offset, relative to whence, to that the lock +starts. whence is as with fileobj.seek(), specifically: 0 - relative to the start of the file (SEEK_SET) 1 - relative to the current buffer position (SEEK_CUR) @@ -437,7 +439,7 @@ starts. `whence` is as with fileobj.seek(), specifically: static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, PyObject *startobj, int whence) -/*[clinic end generated code: output=4985e7a172e7461a input=f666662ec2edd775]*/ +/*[clinic end generated code: output=4985e7a172e7461a input=369bef4d7a1c5ff4]*/ { int ret; int async_err = 0; From 9c3994663b1d8f128f18f53fd4e7871fd0301c4f Mon Sep 17 00:00:00 2001 From: Naitree Zhu Date: Sat, 8 Nov 2025 19:19:42 +0900 Subject: [PATCH 074/417] gh-139741: Make `dist-pdf` docs archive build work for macOS (#140837) --- Doc/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index f6f4c721080..f16d9cacb1b 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -241,7 +241,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" From 87942d911b8bc9e83caee3c0b699f0b0ba15daa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 8 Nov 2025 13:22:02 +0100 Subject: [PATCH 075/417] gh-141004: correctly document `Py_HASH_*` and `PyHASH_*` as `hash_info` attributes (#141233) --- Doc/c-api/hash.rst | 12 +++++++++++- Doc/library/sys.rst | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index ecd604c81bc..1ad712b0ce4 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -56,6 +56,8 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms and 5 on 32-bit platforms should provide a decent safety margin. + This corresponds to the :data:`sys.hash_info.cutoff` constant. + .. versionadded:: 3.4 @@ -63,6 +65,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 @@ -71,7 +74,6 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. - This corresponds to the :data:`sys.hash_info.hash_bits` constant. .. versionadded:: 3.13 @@ -86,6 +88,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. .. versionadded:: 3.13 @@ -94,6 +97,7 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. .. versionadded:: 3.13 @@ -111,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 698a9d0689d..a0621d4b0db 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1176,10 +1176,14 @@ always available. Unless explicitly noted otherwise, all variables are read-only The size of the seed key of the hash algorithm + .. attribute:: hash_info.cutoff + + Cutoff for small string DJBX33A optimization in range ``[1, cutoff)``. + .. versionadded:: 3.2 .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* + Added *algorithm*, *hash_bits*, *seed_bits*, and *cutoff*. .. data:: hexversion From 6545a4e8f83c27996fc771ed7c8c96ae0ce8d2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=AF=E5=87=9B?= <1348292515@qq.com> Date: Sat, 8 Nov 2025 22:56:48 +0800 Subject: [PATCH 076/417] gh-141246: Link to correct Windows docs in `time.sleep()` doc (#141248) --- Doc/library/time.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 69e6433e898..a931134331f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -407,9 +407,9 @@ Functions On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses + continues execution. On Windows 10 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation From be1c72a45d54cdd35e0a830e18224c4c74be808c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 8 Nov 2025 10:47:09 -0500 Subject: [PATCH 077/417] gh-141004: Document `PyErr_ProgramTextObject` and `PyErr_ProgramText` (GH-141250) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index f525ee7a046..c58aa659e1b 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -331,6 +331,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ From 5e5fc0404ed983bb37a19793a5c802d0d9852e5d Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sat, 8 Nov 2025 12:29:31 -0500 Subject: [PATCH 078/417] gh-141004: Document `PyBUF_WRITEABLE` (GH-141255) --- Doc/c-api/buffer.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index d3081894ead..6bb72a2312b 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -261,6 +261,10 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST From 545299773b40fb589cbd5e54d1d597207d9a2a76 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 8 Nov 2025 18:12:03 +0000 Subject: [PATCH 079/417] gh-141004: Document the `PyDoc_VAR` macro (GH-141263) --- Doc/c-api/intro.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index acce3dc215d..4e7d1630ab3 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -235,7 +235,7 @@ complete listing. .. c:macro:: PyDoc_STRVAR(name, str) - Creates a variable with name ``name`` that can be used in docstrings. + Creates a variable with name *name* that can be used in docstrings. If Python is built without docstrings, the value will be empty. Use :c:macro:`PyDoc_STRVAR` for docstrings to support building @@ -267,6 +267,15 @@ complete listing. {NULL, NULL} }; +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given name *name*. + + For example:: + + PyDoc_VAR(python_doc) = PyDoc_STR("A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + .. _api-objects: From 0ac890bea79d3e0162c8909b0999f626f1141d89 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sat, 8 Nov 2025 19:22:05 +0000 Subject: [PATCH 080/417] gh-141004: Document `Py_BUILD_ASSERT*` macros (GH-141266) --- Doc/c-api/intro.rst | 23 +++++++++++++++++++++++ Doc/conf.py | 3 --- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 4e7d1630ab3..6e1a9dcb355 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -233,6 +233,29 @@ complete listing. .. versionadded:: 3.4 +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + + .. versionadded:: 3.3 + +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + .. c:macro:: PyDoc_STRVAR(name, str) Creates a variable with name *name* that can be used in docstrings. diff --git a/Doc/conf.py b/Doc/conf.py index f1dda10052e..0f1412d1007 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -226,9 +226,6 @@ # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. From b36f01d03f8f89cadd8436a7261ca29e4030d30c Mon Sep 17 00:00:00 2001 From: Mohsin Mehmood <55545648+mohsinm-dev@users.noreply.github.com> Date: Sun, 9 Nov 2025 09:49:29 +0500 Subject: [PATCH 081/417] gh-141186: document `asyncio.Task` cancellation propagation behavior (#141249) --- Doc/library/asyncio-task.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f825ae92ec7..863b3e33657 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1221,8 +1221,8 @@ Task Object To cancel a running Task use the :meth:`cancel` method. Calling it will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. + the wrapped coroutine. If a coroutine is awaiting on a future-like + object during cancellation, the awaited object will be cancelled. :meth:`cancelled` can be used to check if the Task was cancelled. The method returns ``True`` if the wrapped coroutine did not @@ -1411,6 +1411,10 @@ Task Object the cancellation, it needs to call :meth:`Task.uncancel` in addition to catching the exception. + If the Task being cancelled is currently awaiting on a future-like + object, that awaited object will also be cancelled. This cancellation + propagates down the entire chain of awaited objects. + .. versionchanged:: 3.9 Added the *msg* parameter. From 7ae440f262c99ba9a3327237f83c9290dc963028 Mon Sep 17 00:00:00 2001 From: Chilla Kalyan <127284726+chillakalyan@users.noreply.github.com> Date: Sun, 9 Nov 2025 14:27:34 +0530 Subject: [PATCH 082/417] gh-141127: Clarify o?s.symlink() documentation for argument order (#141144) --- Doc/library/os.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d31d0ce9c85..dbc3c92c879 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3873,6 +3873,9 @@ features: Create a symbolic link pointing to *src* named *dst*. + The *src* parameter refers to the target of the link (the file or directory being linked to), + and *dst* is the name of the link being created. + On Windows, a symlink represents either a file or a directory, and does not morph to the target dynamically. If the target is present, the type of the symlink will be created to match. Otherwise, the symlink will be created From 5ba0a1aa1fe386fbc863d3fe8f32dfbfe2b1bded Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 9 Nov 2025 12:37:34 +0000 Subject: [PATCH 083/417] gh-136702: Deprecate passing non-ascii *encoding* (str) to `encodings.normalize_encoding` (#140030) Closes #136702 --- Doc/deprecations/pending-removal-in-3.17.rst | 6 ++++++ Lib/email/_header_value_parser.py | 4 ++++ Lib/email/utils.py | 4 ++++ Lib/encodings/__init__.py | 8 +++++++- Lib/test/test_codecs.py | 10 +++++++--- Lib/test/test_email/test_email.py | 3 ++- Lib/test/test_email/test_headerregistry.py | 10 +++++++++- .../2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst | 3 +++ 8 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 0a1c2f08cab..e769c9d371e 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -23,6 +23,12 @@ Pending removal in Python 3.17 (Contributed by Shantanu Jain in :gh:`91896`.) +* :mod:`encodings`: + + - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` + is deprecated and scheduled for removal in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`136702`) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 91243378dc0..c7f665b3990 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -796,6 +796,10 @@ def params(self): value = urllib.parse.unquote(value, encoding='latin-1') else: try: + # Explicitly look up the codec for warning generation, see gh-140030 + # Can be removed in 3.17 + import codecs + codecs.lookup(charset) value = value.decode(charset, 'surrogateescape') except (LookupError, UnicodeEncodeError): # XXX: there should really be a custom defect for diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 3de1f0d24a1..d4824dc3601 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -460,6 +460,10 @@ def collapse_rfc2231_value(value, errors='replace', charset = fallback_charset rawbytes = bytes(text, 'raw-unicode-escape') try: + # Explicitly look up the codec for warning generation, see gh-140030 + # Can be removed in 3.17 + import codecs + codecs.lookup(charset) return str(rawbytes, charset, errors) except LookupError: # charset is not a known codec. diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index e7e4ca3358e..e205ec32637 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -26,7 +26,7 @@ (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. -"""#" +""" import codecs import sys @@ -56,6 +56,12 @@ def normalize_encoding(encoding): if isinstance(encoding, bytes): encoding = str(encoding, "ascii") + if not encoding.isascii(): + import warnings + warnings.warn( + "Support for non-ascii encoding names will be removed in 3.17", + DeprecationWarning, stacklevel=2) + return _normalize_encoding(encoding) def search_function(encoding): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c35a4508943..f1f0ac5ad36 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3886,15 +3886,14 @@ def search_function(encoding): self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa-8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA---8'), ('test.aaa---8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa---8', 2, 3, 4)) - self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA.8'), ('test.aaa.8', 2, 3, 4)) self.assertEqual(codecs.lookup('TEST.AAA...8'), ('test.aaa...8', 2, 3, 4)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) def test_encodings_normalize_encoding(self): - # encodings.normalize_encoding() ignores non-ASCII characters. normalize = encodings.normalize_encoding self.assertEqual(normalize('utf_8'), 'utf_8') - self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') self.assertEqual(normalize('utf 8'), 'utf_8') # encodings.normalize_encoding() doesn't convert # characters to lower case. @@ -3902,6 +3901,11 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf.8'), 'utf.8') self.assertEqual(normalize('utf...8'), 'utf...8') + # Non-ASCII *encoding* is deprecated. + with self.assertWarnsRegex(DeprecationWarning, + "Support for non-ascii encoding names will be removed in 3.17"): + self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 4cd587bcd76..1900adf463b 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -5738,7 +5738,8 @@ def test_rfc2231_bad_character_in_encoding(self): """ msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), 'myfile.txt') + with self.assertWarns(DeprecationWarning): + self.assertEqual(msg.get_filename(), 'myfile.txt') def test_rfc2231_single_tick_in_filename_extended(self): eq = self.assertEqual diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index ff7a6da644d..1d0d0a49a82 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -247,7 +247,15 @@ def content_type_as_value(self, decoded = args[2] if l>2 and args[2] is not DITTO else source header = 'Content-Type:' + ' ' if source else '' folded = args[3] if l>3 else header + decoded + '\n' - h = self.make_header('Content-Type', source) + # Both rfc2231 test cases with utf-8%E2%80%9D raise warnings, + # clear encoding cache to ensure test isolation. + if 'utf-8%E2%80%9D' in source and 'ascii' not in source: + import encodings + encodings._cache.clear() + with self.assertWarns(DeprecationWarning): + h = self.make_header('Content-Type', source) + else: + h = self.make_header('Content-Type', source) self.assertEqual(h.content_type, content_type) self.assertEqual(h.maintype, maintype) self.assertEqual(h.subtype, subtype) diff --git a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst new file mode 100644 index 00000000000..88303f017f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst @@ -0,0 +1,3 @@ +:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to +:func:`encodings.normalize_encoding` and schedule removal of support for +Python 3.17. From 0c77e7c23b5c270a3142105542c56c59b59c52a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 13:41:08 +0100 Subject: [PATCH 084/417] gh-140530: fix a reference leak in an error path for `raise exc from cause` (#140908) Fix a reference leak in `raise E from T` when `T` is an exception subtype for which `T.__new__` does not return an exception instance. --- Lib/test/test_raise.py | 18 +++++++----------- ...5-11-02-12-47-38.gh-issue-140530.S934bp.rst | 2 ++ Python/ceval.c | 1 + 3 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index dcf0753bc82..645ef291a58 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -186,18 +186,14 @@ def test_class_cause(self): self.fail("No exception raised") def test_class_cause_nonexception_result(self): - class ConstructsNone(BaseException): - @classmethod + # See https://github.com/python/cpython/issues/140530. + class ConstructMortal(BaseException): def __new__(*args, **kwargs): - return None - try: - raise IndexError from ConstructsNone - except TypeError as e: - self.assertIn("should have returned an instance of BaseException", str(e)) - except IndexError: - self.fail("Wrong kind of exception raised") - else: - self.fail("No exception raised") + return ["mortal value"] + + msg = ".*should have returned an instance of BaseException.*" + with self.assertRaisesRegex(TypeError, msg): + raise IndexError from ConstructMortal def test_instance_cause(self): cause = KeyError() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst new file mode 100644 index 00000000000..e3af493893a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst @@ -0,0 +1,2 @@ +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. diff --git a/Python/ceval.c b/Python/ceval.c index 7ca7ae48b5c..43e8ee71206 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2148,6 +2148,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) "calling %R should have returned an instance of " "BaseException, not %R", cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); goto raise_error; } Py_DECREF(cause); From 3ce2d57b2f02030353af314d89c5f6215d2f5c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 13:45:38 +0100 Subject: [PATCH 085/417] gh-100218: correctly set `errno` when `socket.if_{nametoindex,indextoname}` raise `OSError` (#140905) Previously, socket.if_nametoindex() and socket.if_indextoname() could raise an `OSError` with a `None` errno. Now, the errno from libc is propagated. --- Lib/test/test_socket.py | 10 ++++++++-- .../2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst | 3 +++ Modules/socketmodule.c | 5 +++-- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 24ee0f2c280..934b7137096 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1176,7 +1176,10 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): - self.assertRaises(OSError, socket.if_indextoname, 0) + with self.assertRaises(OSError) as cm: + socket.if_indextoname(0) + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(ValueError, socket.if_indextoname, -1) self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') @@ -1196,8 +1199,11 @@ def testInvalidInterfaceIndexToName(self): 'socket.if_nametoindex() not available.') @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): + with self.assertRaises(OSError) as cm: + socket.if_nametoindex("_DEADBEEF") + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(TypeError, socket.if_nametoindex, 0) - self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @unittest.skipUnless(hasattr(sys, 'getrefcount'), 'test needs sys.getrefcount()') diff --git a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst new file mode 100644 index 00000000000..2f7500d2955 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst @@ -0,0 +1,3 @@ +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index dd4b6892977..6a844d44bf0 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7294,10 +7294,10 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) unsigned long index; #endif + errno = ENODEV; // in case 'if_nametoindex' does not set errno index = if_nametoindex(PyBytes_AS_STRING(oname)); if (index == 0) { - /* if_nametoindex() doesn't set errno */ - PyErr_SetString(PyExc_OSError, "no interface with this name"); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -7317,6 +7317,7 @@ static PyObject * _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) /*[clinic end generated code: output=e48bc324993052e0 input=c93f753d0cf6d7d1]*/ { + errno = ENXIO; // in case 'if_indextoname' does not set errno char name[IF_NAMESIZE + 1]; if (if_indextoname(index, name) == NULL) { PyErr_SetFromErrno(PyExc_OSError); From ae1f435071148571422e19be598336726eb25b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:14:08 +0100 Subject: [PATCH 086/417] gh-111389: replace deprecated occurrences of `_PyHASH_*` macros (#141236) --- Include/cpython/pyhash.h | 4 ++-- Modules/_decimal/_decimal.c | 6 +++--- Objects/complexobject.c | 2 +- Objects/longobject.c | 38 ++++++++++++++++++------------------- Python/pyhash.c | 26 ++++++++++++------------- Python/sysmodule.c | 6 +++--- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index a33ba10b8d3..dac223368db 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -7,7 +7,7 @@ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ + reduction modulo the prime 2**PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 # define PyHASH_BITS 61 @@ -15,7 +15,7 @@ # define PyHASH_BITS 31 #endif -#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_MODULUS (((size_t)1 << PyHASH_BITS) - 1) #define PyHASH_INF 314159 #define PyHASH_IMAG PyHASH_MULTIPLIER diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 44917ed7357..0484d9896a1 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -5799,7 +5799,7 @@ _decimal_Decimal___floor___impl(PyObject *self, PyTypeObject *cls) static Py_hash_t _dec_hash(PyDecObject *v) { -#if defined(CONFIG_64) && _PyHASH_BITS == 61 +#if defined(CONFIG_64) && PyHASH_BITS == 61 /* 2**61 - 1 */ mpd_uint_t p_data[1] = {2305843009213693951ULL}; mpd_t p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 19, 1, 1, p_data}; @@ -5807,7 +5807,7 @@ _dec_hash(PyDecObject *v) mpd_uint_t inv10_p_data[1] = {2075258708292324556ULL}; mpd_t inv10_p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 19, 1, 1, inv10_p_data}; -#elif defined(CONFIG_32) && _PyHASH_BITS == 31 +#elif defined(CONFIG_32) && PyHASH_BITS == 31 /* 2**31 - 1 */ mpd_uint_t p_data[2] = {147483647UL, 2}; mpd_t p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 10, 2, 2, p_data}; @@ -5816,7 +5816,7 @@ _dec_hash(PyDecObject *v) mpd_t inv10_p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 10, 2, 2, inv10_p_data}; #else - #error "No valid combination of CONFIG_64, CONFIG_32 and _PyHASH_BITS" + #error "No valid combination of CONFIG_64, CONFIG_32 and PyHASH_BITS" #endif const Py_hash_t py_hash_inf = 314159; mpd_uint_t ten_data[1] = {10}; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 03fc137c345..6247376a0e6 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -644,7 +644,7 @@ complex_hash(PyObject *op) * compare equal must have the same hash value, so that * hash(x + 0*j) must equal hash(x). */ - combined = hashreal + _PyHASH_IMAG * hashimag; + combined = hashreal + PyHASH_IMAG * hashimag; if (combined == (Py_uhash_t)-1) combined = (Py_uhash_t)-2; return (Py_hash_t)combined; diff --git a/Objects/longobject.c b/Objects/longobject.c index 298399210af..43c0db753a0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3703,36 +3703,36 @@ long_hash(PyObject *obj) #endif while (--i >= 0) { - /* Here x is a quantity in the range [0, _PyHASH_MODULUS); we + /* Here x is a quantity in the range [0, PyHASH_MODULUS); we want to compute x * 2**PyLong_SHIFT + v->long_value.ob_digit[i] modulo - _PyHASH_MODULUS. + PyHASH_MODULUS. - The computation of x * 2**PyLong_SHIFT % _PyHASH_MODULUS + The computation of x * 2**PyLong_SHIFT % PyHASH_MODULUS amounts to a rotation of the bits of x. To see this, write - x * 2**PyLong_SHIFT = y * 2**_PyHASH_BITS + z + x * 2**PyLong_SHIFT = y * 2**PyHASH_BITS + z - where y = x >> (_PyHASH_BITS - PyLong_SHIFT) gives the top + where y = x >> (PyHASH_BITS - PyLong_SHIFT) gives the top PyLong_SHIFT bits of x (those that are shifted out of the - original _PyHASH_BITS bits, and z = (x << PyLong_SHIFT) & - _PyHASH_MODULUS gives the bottom _PyHASH_BITS - PyLong_SHIFT - bits of x, shifted up. Then since 2**_PyHASH_BITS is - congruent to 1 modulo _PyHASH_MODULUS, y*2**_PyHASH_BITS is - congruent to y modulo _PyHASH_MODULUS. So + original PyHASH_BITS bits, and z = (x << PyLong_SHIFT) & + PyHASH_MODULUS gives the bottom PyHASH_BITS - PyLong_SHIFT + bits of x, shifted up. Then since 2**PyHASH_BITS is + congruent to 1 modulo PyHASH_MODULUS, y*2**PyHASH_BITS is + congruent to y modulo PyHASH_MODULUS. So - x * 2**PyLong_SHIFT = y + z (mod _PyHASH_MODULUS). + x * 2**PyLong_SHIFT = y + z (mod PyHASH_MODULUS). The right-hand side is just the result of rotating the - _PyHASH_BITS bits of x left by PyLong_SHIFT places; since - not all _PyHASH_BITS bits of x are 1s, the same is true - after rotation, so 0 <= y+z < _PyHASH_MODULUS and y + z is + PyHASH_BITS bits of x left by PyLong_SHIFT places; since + not all PyHASH_BITS bits of x are 1s, the same is true + after rotation, so 0 <= y+z < PyHASH_MODULUS and y + z is the reduction of x*2**PyLong_SHIFT modulo - _PyHASH_MODULUS. */ - x = ((x << PyLong_SHIFT) & _PyHASH_MODULUS) | - (x >> (_PyHASH_BITS - PyLong_SHIFT)); + PyHASH_MODULUS. */ + x = ((x << PyLong_SHIFT) & PyHASH_MODULUS) | + (x >> (PyHASH_BITS - PyLong_SHIFT)); x += v->long_value.ob_digit[i]; - if (x >= _PyHASH_MODULUS) - x -= _PyHASH_MODULUS; + if (x >= PyHASH_MODULUS) + x -= PyHASH_MODULUS; } x = x * sign; if (x == (Py_uhash_t)-1) diff --git a/Python/pyhash.c b/Python/pyhash.c index 216f437dd9a..157312a936b 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -29,7 +29,7 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; #endif /* For numeric types, the hash of a number x is based on the reduction - of x modulo the prime P = 2**_PyHASH_BITS - 1. It's designed so that + of x modulo the prime P = 2**PyHASH_BITS - 1. It's designed so that hash(x) == hash(y) whenever x and y are numerically equal, even if x and y have different types. @@ -52,8 +52,8 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; If the result of the reduction is infinity (this is impossible for integers, floats and Decimals) then use the predefined hash value - _PyHASH_INF for x >= 0, or -_PyHASH_INF for x < 0, instead. - _PyHASH_INF and -_PyHASH_INF are also used for the + PyHASH_INF for x >= 0, or -PyHASH_INF for x < 0, instead. + PyHASH_INF and -PyHASH_INF are also used for the hashes of float and Decimal infinities. NaNs hash with a pointer hash. Having distinct hash values prevents @@ -65,16 +65,16 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; efficiently, even if the exponent of the binary or decimal number is large. The key point is that - reduce(x * y) == reduce(x) * reduce(y) (modulo _PyHASH_MODULUS) + reduce(x * y) == reduce(x) * reduce(y) (modulo PyHASH_MODULUS) provided that {reduce(x), reduce(y)} != {0, infinity}. The reduction of a binary or decimal float is never infinity, since the denominator is a power of 2 (for binary) or a divisor of a power of 10 (for decimal). So we have, for nonnegative x, - reduce(x * 2**e) == reduce(x) * reduce(2**e) % _PyHASH_MODULUS + reduce(x * 2**e) == reduce(x) * reduce(2**e) % PyHASH_MODULUS - reduce(x * 10**e) == reduce(x) * reduce(10**e) % _PyHASH_MODULUS + reduce(x * 10**e) == reduce(x) * reduce(10**e) % PyHASH_MODULUS and reduce(10**e) can be computed efficiently by the usual modular exponentiation algorithm. For reduce(2**e) it's even better: since @@ -92,7 +92,7 @@ _Py_HashDouble(PyObject *inst, double v) if (!isfinite(v)) { if (isinf(v)) - return v > 0 ? _PyHASH_INF : -_PyHASH_INF; + return v > 0 ? PyHASH_INF : -PyHASH_INF; else return PyObject_GenericHash(inst); } @@ -109,19 +109,19 @@ _Py_HashDouble(PyObject *inst, double v) and hexadecimal floating point. */ x = 0; while (m) { - x = ((x << 28) & _PyHASH_MODULUS) | x >> (_PyHASH_BITS - 28); + x = ((x << 28) & PyHASH_MODULUS) | x >> (PyHASH_BITS - 28); m *= 268435456.0; /* 2**28 */ e -= 28; y = (Py_uhash_t)m; /* pull out integer part */ m -= y; x += y; - if (x >= _PyHASH_MODULUS) - x -= _PyHASH_MODULUS; + if (x >= PyHASH_MODULUS) + x -= PyHASH_MODULUS; } - /* adjust for the exponent; first reduce it modulo _PyHASH_BITS */ - e = e >= 0 ? e % _PyHASH_BITS : _PyHASH_BITS-1-((-1-e) % _PyHASH_BITS); - x = ((x << e) & _PyHASH_MODULUS) | x >> (_PyHASH_BITS - e); + /* adjust for the exponent; first reduce it modulo PyHASH_BITS */ + e = e >= 0 ? e % PyHASH_BITS : PyHASH_BITS-1-((-1-e) % PyHASH_BITS); + x = ((x << e) & PyHASH_MODULUS) | x >> (PyHASH_BITS - e); x = x * sign; if (x == (Py_uhash_t)-1) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 86dd1395cae..a611844f76e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1587,10 +1587,10 @@ get_hash_info(PyThreadState *tstate) } while(0) SET_HASH_INFO_ITEM(PyLong_FromLong(8 * sizeof(Py_hash_t))); - SET_HASH_INFO_ITEM(PyLong_FromSsize_t(_PyHASH_MODULUS)); - SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_INF)); + SET_HASH_INFO_ITEM(PyLong_FromSsize_t(PyHASH_MODULUS)); + SET_HASH_INFO_ITEM(PyLong_FromLong(PyHASH_INF)); SET_HASH_INFO_ITEM(PyLong_FromLong(0)); // This is no longer used - SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_IMAG)); + SET_HASH_INFO_ITEM(PyLong_FromLong(PyHASH_IMAG)); SET_HASH_INFO_ITEM(PyUnicode_FromString(hashfunc->name)); SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->hash_bits)); SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->seed_bits)); From 1d738dea6364de004f8cec7c6309d6bbd3b996c7 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 10:06:38 -0500 Subject: [PATCH 087/417] gh-141004: Document deprecated aliases for memory allocation (GH-141146) --- Doc/c-api/allocation.rst | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59d913a0462..59044d2d88c 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -140,10 +140,6 @@ Allocating Objects on the Heap * :c:member:`~PyTypeObject.tp_alloc` -.. c:function:: void PyObject_Del(void *op) - - Same as :c:func:`PyObject_Free`. - .. c:var:: PyObject _Py_NoneStruct Object which is visible in Python as ``None``. This should only be accessed @@ -156,3 +152,35 @@ Allocating Objects on the Heap :ref:`moduleobjects` To allocate and create extension modules. + +Deprecated aliases +^^^^^^^^^^^^^^^^^^ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` From 60155329a0a83a2b9e740f0c0de41c9d44f5a053 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:32:39 +0000 Subject: [PATCH 088/417] gh-141004: Document `PyWeakref_CheckRefExact` (GH-141279) --- Doc/c-api/weakref.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 14ec9d951c4..39e4febd3ef 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) From dbe40904a78a0c8ffa25fb64e6ff1e14e6e7ba5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Nov 2025 16:44:26 +0100 Subject: [PATCH 089/417] gh-141004: document `curses` C API (#141254) --- Doc/c-api/concrete.rst | 10 ++- Doc/c-api/curses.rst | 138 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 Doc/c-api/curses.rst diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce6..a5c5a53236c 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -115,5 +115,13 @@ Other Objects gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 00000000000..5a1697c43cc --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + From 8dd849828636bb3989c6d5d20f8790a3fb770fc4 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 11:21:44 -0500 Subject: [PATCH 090/417] gh-141004: Document `Py_func_type_input` (GH-141273) --- Doc/c-api/veryhigh.rst | 87 ++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 0b2b55b6387..916c616dfee 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -183,8 +184,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start symbol and must one of the following: - :c:data:`Py_eval_input`, :c:data:`Py_file_input`, or :c:data:`Py_single_input`. + the start symbol and must one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -233,8 +233,8 @@ the same library that the Python runtime is using. Parse and compile the Python source code in *str*, returning the resulting code object. The start symbol is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -297,32 +297,6 @@ the same library that the Python runtime is using. true on success, false on failure. -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - .. c:struct:: PyCompilerFlags This is the structure used to hold compiler flags. In cases where code is only @@ -366,3 +340,52 @@ the same library that the Python runtime is using. as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:var:: int Py_eval_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for isolated expressions; for use with + :c:func:`Py_CompileString`. + + +.. c:var:: int Py_file_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for sequences of statements as read + from a file or other source; for use with :c:func:`Py_CompileString`. This is + the symbol to use when compiling arbitrarily long Python source code. + + +.. c:var:: int Py_single_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a single statement; for use with + :c:func:`Py_CompileString`. This is the symbol used for the interactive + interpreter loop. + + +.. c:var:: int Py_func_type_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. + + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. + + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` + + .. versionadded:: 3.8 From b5a0c72492800c7e999b87adfcfeabaacb4ecb97 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 12:09:09 -0500 Subject: [PATCH 091/417] gh-141004: Document `PyExceptionInstance*` APIs (GH-141301) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/exceptions.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c58aa659e1b..5241533e112 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -788,6 +788,17 @@ Exception Classes Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as From 18529b580b59b8d075641da6c685bef377eb0a7b Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 12:49:17 -0500 Subject: [PATCH 092/417] gh-141004: Document `PyFunction_SetKwDefaults` (GH-141294) --- Doc/c-api/function.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 0bac6389571..609b5e885b6 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -102,6 +102,15 @@ There are a few functions specific to Python functions. dictionary of arguments or ``NULL``. +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` From 807db68ddd8572cfa825373bc13461b02691f4d9 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 9 Nov 2025 13:03:38 -0500 Subject: [PATCH 093/417] gh-141004: Document `PyClassMethod*` and `PyStaticMethod*` APIs (GH-141296) --- Doc/c-api/descriptor.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index ff0df575279..9a4093a7708 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -38,3 +38,39 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) + + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + From 6f20ea1e2d302b7b88d64b6786abbad1747ff950 Mon Sep 17 00:00:00 2001 From: Lakshya Upadhyaya Date: Mon, 10 Nov 2025 00:29:06 +0530 Subject: [PATCH 094/417] gh-140980: document `SET_FUNCTION_ATTRIBUTE` flag for `annotate` function (#141306) --- Doc/library/dis.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 284eeff5e4d..a24589fd0a5 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1673,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) From 14c62227f9fa11fb743f9e03dcc5aab553de1098 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 9 Nov 2025 19:53:56 +0000 Subject: [PATCH 095/417] gh-141004: Document `PySuper_Type` (GH-141315) --- Doc/c-api/descriptor.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 9a4093a7708..22c3b790cc3 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -43,6 +43,12 @@ found in the dictionary of type objects. Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + .. c:var:: PyTypeObject PyClassMethod_Type The type of class method objects. This is the same object as From ec85d3cbfe315086805c33bb64c28a8509098829 Mon Sep 17 00:00:00 2001 From: Elena O <31424287+oklena@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:42:22 -0800 Subject: [PATCH 096/417] gh-62480: De-personalize "Mocking Unbound Methods" section in `unittest.mock` examples (#141322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Rewrite Mocking Unbound Methods paragraph to second person Co-authored-by: C.A.M. Gerlach Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/unittest.mock-examples.rst | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 00cc9bfc0a5..e2b0322dae0 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -743,16 +743,15 @@ exception is raised in the setUp then tearDown is not called. Mocking Unbound Methods ~~~~~~~~~~~~~~~~~~~~~~~ -Whilst writing tests today I needed to patch an *unbound method* (patching the -method on the class rather than on the instance). I needed self to be passed -in as the first argument because I want to make asserts about which objects -were calling this particular method. The issue is that you can't patch with a -mock for this, because if you replace an unbound method with a mock it doesn't -become a bound method when fetched from the instance, and so it doesn't get -self passed in. The workaround is to patch the unbound method with a real -function instead. The :func:`patch` decorator makes it so simple to -patch out methods with a mock that having to create a real function becomes a -nuisance. +Sometimes a test needs to patch an *unbound method*, which means patching the +method on the class rather than on the instance. In order to make assertions +about which objects were calling this particular method, you need to pass +``self`` as the first argument. The issue is that you can't patch with a mock for +this, because if you replace an unbound method with a mock it doesn't become +a bound method when fetched from the instance, and so it doesn't get ``self`` +passed in. The workaround is to patch the unbound method with a real function +instead. The :func:`patch` decorator makes it so simple to patch out methods +with a mock that having to create a real function becomes a nuisance. If you pass ``autospec=True`` to patch then it does the patching with a *real* function object. This function object has the same signature as the one @@ -760,8 +759,8 @@ it is replacing, but delegates to a mock under the hood. You still get your mock auto-created in exactly the same way as before. What it means though, is that if you use it to patch out an unbound method on a class the mocked function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I -wanted: +It will have ``self`` passed in as the first argument, which is exactly what +was needed: >>> class Foo: ... def foo(self): From b618731781c31d4b5b75d199dfc14993ffd66e37 Mon Sep 17 00:00:00 2001 From: KarnbirKhera <166065758+KarnbirKhera@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:45:38 -0800 Subject: [PATCH 097/417] gh-62480: De-personalize "Partial mocking" section in `unittest.mock` examples (#141321) * Refine some wording in unittest partial mock doc Some of the descriptions were addressed in first person, but have now been changed to address the user reading the documentation instead. Co-authored-by: C.A.M. Gerlach --- Doc/library/unittest.mock-examples.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index e2b0322dae0..6af4298d44f 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -600,13 +600,13 @@ this list of calls for us:: Partial mocking ~~~~~~~~~~~~~~~ -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. +For some tests, you may want to mock out a call to :meth:`datetime.date.today` +to return a known date, but don't want to prevent the code under test from +creating new date objects. Unfortunately :class:`datetime.date` is written in C, +so you cannot just monkey-patch out the static :meth:`datetime.date.today` method. -I found a simple way of doing this that involved effectively wrapping the date -class with a mock, but passing through calls to the constructor to the real +Instead, you can effectively wrap the date +class with a mock, while passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to From 9b0179fa87fee39df0f75bd84fc2dd75f1d00553 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 00:43:03 +0000 Subject: [PATCH 098/417] gh-141004: Document `Py_DTSF_*` macros (GH-141310) --- Doc/c-api/conversion.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index cc7a3d9d956..e9d866c647d 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -128,18 +128,28 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + Always precede the returned string with a sign + character, even if *val* is non-negative. - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 If *ptype* is non-``NULL``, then the value it points to will be set to one of ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that From df192616212f80aaa2f672b722b925943dbd3b78 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:05:06 +0000 Subject: [PATCH 099/417] gh-141004: Document `pyctype.h` macros (GH-141272) --- Doc/c-api/conversion.rst | 55 ++++++++++++++++++++++++++++++++++++++++ Doc/library/locale.rst | 4 +-- Doc/whatsnew/2.7.rst | 18 ++++++------- 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index e9d866c647d..533e5460da8 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -172,3 +172,58 @@ The following functions provide locale-independent string to number conversions. Case insensitive comparison of strings. The function works almost identically to :c:func:`!strncmp` except that it ignores the case. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) + + Return the lowercase equivalent of the character *c*. + + +.. c:macro:: Py_TOUPPER(c) + + Return the uppercase equivalent of the character *c*. diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 4824391e597..94fc046d3f3 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -524,8 +524,8 @@ The :mod:`locale` module defines the following exception and functions: SSH connections. Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. + from ``ctype.h``. Instead, ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`Py_TOLOWER`. .. data:: LC_COLLATE diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 09feb185b82..7296296d144 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2181,14 +2181,14 @@ Changes to Python's build process and to the C API include: discussed in :issue:`5753`, and fixed by Antoine Pitrou. * New macros: the Python header files now define the following macros: - :c:macro:`!Py_ISALNUM`, - :c:macro:`!Py_ISALPHA`, - :c:macro:`!Py_ISDIGIT`, - :c:macro:`!Py_ISLOWER`, - :c:macro:`!Py_ISSPACE`, - :c:macro:`!Py_ISUPPER`, - :c:macro:`!Py_ISXDIGIT`, - :c:macro:`!Py_TOLOWER`, and :c:macro:`!Py_TOUPPER`. + :c:macro:`Py_ISALNUM`, + :c:macro:`Py_ISALPHA`, + :c:macro:`Py_ISDIGIT`, + :c:macro:`Py_ISLOWER`, + :c:macro:`Py_ISSPACE`, + :c:macro:`Py_ISUPPER`, + :c:macro:`Py_ISXDIGIT`, + :c:macro:`Py_TOLOWER`, and :c:macro:`Py_TOUPPER`. All of these functions are analogous to the C standard macros for classifying characters, but ignore the current locale setting, because in @@ -2196,8 +2196,6 @@ Changes to Python's build process and to the C API include: locale-independent way. (Added by Eric Smith; :issue:`5793`.) - .. XXX these macros don't seem to be described in the c-api docs. - * Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be From 6d710a79eaf085bf826c6f147e813495519a0897 Mon Sep 17 00:00:00 2001 From: Karina Souza <97332018+KarinaS0uza@users.noreply.github.com> Date: Mon, 10 Nov 2025 04:39:49 -0800 Subject: [PATCH 100/417] gh-140500: Update download.html instructions (#141320) Co-authored-by: Joseph Anthony Pasquale Holsten --- Doc/tools/templates/download.html | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index f914ad86211..c78c650b1cb 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -31,8 +31,7 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} -

{% trans %}To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.{% endtrans %}

+

{% trans %}Download an archive containing all the documentation for this version of Python:{% endtrans %}

@@ -62,8 +61,6 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

-

{% trans %}These archives contain all the content in the documentation.{% endtrans %}

-

{% trans %} We no longer provide pre-built PDFs of the documentation. To build a PDF archive, follow the instructions in the @@ -75,7 +72,6 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

See the directory listing for file sizes.{% endtrans %}

-

{% trans %}Problems{% endtrans %}

{% set bugs = pathto('bugs') %}

{% trans bugs = bugs %}Open an issue From 13fa313bebed71d8bc64f1cfdaf4b2f1ddd3ce5f Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:37:34 +0000 Subject: [PATCH 101/417] gh-139707: Specify `winreg`, `msvcrt` and `winsound` module availability in docs (GH-140429) --- Doc/library/msvcrt.rst | 2 ++ Doc/library/winreg.rst | 2 ++ Doc/library/winsound.rst | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 327cc3602b1..a2c5e375d2c 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -22,6 +22,8 @@ api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where ever possible. +.. availability:: Windows. + .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` was raised. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index df8fb83a018..b150c53735d 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -14,6 +14,8 @@ integer as the registry handle, a :ref:`handle object ` is used to ensure that the handles are closed correctly, even if the programmer neglects to explicitly close them. +.. availability:: Windows. + .. _exception-changed: .. versionchanged:: 3.3 diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 925984c3cdb..93c0c025982 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -13,6 +13,8 @@ The :mod:`winsound` module provides access to the basic sound-playing machinery provided by Windows platforms. It includes functions and several constants. +.. availability:: Windows. + .. function:: Beep(frequency, duration) From 9f5152441d32166134c3c64f56f974b9476f9478 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 10 Nov 2025 14:42:18 +0100 Subject: [PATCH 102/417] gh-136702: Clear codec caches for refleak tests; use test.support helpers (GH-141345) This should fix refleak buildbots. --- Lib/test/libregrtest/utils.py | 19 +++++++++++++++++++ Lib/test/test_codecs.py | 5 +++-- Lib/test/test_email/test_email.py | 3 ++- Lib/test/test_email/test_headerregistry.py | 3 ++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index d94fb84a743..cfb009c203e 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -294,6 +294,25 @@ def clear_caches(): else: importlib_metadata.FastPath.__new__.cache_clear() + try: + encodings = sys.modules['encodings'] + except KeyError: + pass + else: + encodings._cache.clear() + + try: + codecs = sys.modules['codecs'] + except KeyError: + pass + else: + # There's no direct API to clear the codecs search cache, but + # `unregister` clears it implicitly. + def noop_search_function(name): + return None + codecs.register(noop_search_function) + codecs.unregister(noop_search_function) + def get_build_info(): # Get most important configure and build options as a list of strings. diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index f1f0ac5ad36..c31faec9ee5 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -13,6 +13,7 @@ from test import support from test.support import os_helper +from test.support import warnings_helper try: import _testlimitedcapi @@ -3902,8 +3903,8 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf...8'), 'utf...8') # Non-ASCII *encoding* is deprecated. - with self.assertWarnsRegex(DeprecationWarning, - "Support for non-ascii encoding names will be removed in 3.17"): + msg = "Support for non-ascii encoding names will be removed in 3.17" + with warnings_helper.check_warnings((msg, DeprecationWarning)): self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 1900adf463b..4020f1041c4 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -41,6 +41,7 @@ from test import support from test.support import threading_helper +from test.support import warnings_helper from test.support.os_helper import unlink from test.test_email import openfile, TestEmailBase @@ -5738,7 +5739,7 @@ def test_rfc2231_bad_character_in_encoding(self): """ msg = email.message_from_string(m) - with self.assertWarns(DeprecationWarning): + with warnings_helper.check_warnings(('', DeprecationWarning)): self.assertEqual(msg.get_filename(), 'myfile.txt') def test_rfc2231_single_tick_in_filename_extended(self): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 1d0d0a49a82..7138aa4c556 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -8,6 +8,7 @@ from email import headerregistry from email.headerregistry import Address, Group from test.support import ALWAYS_EQ +from test.support import warnings_helper DITTO = object() @@ -252,7 +253,7 @@ def content_type_as_value(self, if 'utf-8%E2%80%9D' in source and 'ascii' not in source: import encodings encodings._cache.clear() - with self.assertWarns(DeprecationWarning): + with warnings_helper.check_warnings(('', DeprecationWarning)): h = self.make_header('Content-Type', source) else: h = self.make_header('Content-Type', source) From 12837c63635559873a5abddf511d38456d69617b Mon Sep 17 00:00:00 2001 From: David Ellis Date: Mon, 10 Nov 2025 13:57:11 +0000 Subject: [PATCH 103/417] gh-137530: generate an __annotate__ function for dataclasses __init__ (GH-137711) --- Doc/whatsnew/3.15.rst | 8 + Lib/dataclasses.py | 94 ++++++++++-- Lib/test/test_dataclasses/__init__.py | 139 +++++++++++++++++- ...-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst | 1 + 4 files changed, 227 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 5379ac3abba..e0b0471567c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -368,6 +368,14 @@ collections.abc previously emitted if it was merely imported or accessed from the :mod:`!collections.abc` module. + +dataclasses +----------- + +* Annotations for generated ``__init__`` methods no longer include internal + type names. + + dbm --- diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b98f21dcbe9..3ccb7246928 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -441,9 +441,11 @@ def __init__(self, globals): self.locals = {} self.overwrite_errors = {} self.unconditional_adds = {} + self.method_annotations = {} def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, - overwrite_error=False, unconditional_add=False, decorator=None): + overwrite_error=False, unconditional_add=False, decorator=None, + annotation_fields=None): if locals is not None: self.locals.update(locals) @@ -464,16 +466,14 @@ def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, self.names.append(name) - if return_type is not MISSING: - self.locals[f'__dataclass_{name}_return_type__'] = return_type - return_annotation = f'->__dataclass_{name}_return_type__' - else: - return_annotation = '' + if annotation_fields is not None: + self.method_annotations[name] = (annotation_fields, return_type) + args = ','.join(args) body = '\n'.join(body) # Compute the text of the entire function, add it to the text we're generating. - self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}):\n{body}') def add_fns_to_class(self, cls): # The source to all of the functions we're generating. @@ -509,6 +509,15 @@ def add_fns_to_class(self, cls): # Now that we've generated the functions, assign them into cls. for name, fn in zip(self.names, fns): fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + + try: + annotation_fields, return_type = self.method_annotations[name] + except KeyError: + pass + else: + annotate_fn = _make_annotate_function(cls, name, annotation_fields, return_type) + fn.__annotate__ = annotate_fn + if self.unconditional_adds.get(name, False): setattr(cls, name, fn) else: @@ -524,6 +533,44 @@ def add_fns_to_class(self, cls): raise TypeError(error_msg) +def _make_annotate_function(__class__, method_name, annotation_fields, return_type): + # Create an __annotate__ function for a dataclass + # Try to return annotations in the same format as they would be + # from a regular __init__ function + + def __annotate__(format, /): + Format = annotationlib.Format + match format: + case Format.VALUE | Format.FORWARDREF | Format.STRING: + cls_annotations = {} + for base in reversed(__class__.__mro__): + cls_annotations.update( + annotationlib.get_annotations(base, format=format) + ) + + new_annotations = {} + for k in annotation_fields: + new_annotations[k] = cls_annotations[k] + + if return_type is not MISSING: + if format == Format.STRING: + new_annotations["return"] = annotationlib.type_repr(return_type) + else: + new_annotations["return"] = return_type + + return new_annotations + + case _: + raise NotImplementedError(format) + + # This is a flag for _add_slots to know it needs to regenerate this method + # In order to remove references to the original class when it is replaced + __annotate__.__generated_by_dataclasses__ = True + __annotate__.__qualname__ = f"{__class__.__qualname__}.{method_name}.__annotate__" + + return __annotate__ + + def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple @@ -612,7 +659,7 @@ def _init_param(f): elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=__dataclass_HAS_DEFAULT_FACTORY__' - return f'{f.name}:__dataclass_type_{f.name}__{default}' + return f'{f.name}{default}' def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, @@ -635,11 +682,10 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, - **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - } - } + annotation_fields = [f.name for f in fields if f.init] + + locals = {'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object} body_lines = [] for f in fields: @@ -670,7 +716,8 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, [self_name] + _init_params, body_lines, locals=locals, - return_type=None) + return_type=None, + annotation_fields=annotation_fields) def _frozen_get_del_attr(cls, fields, func_builder): @@ -1337,6 +1384,25 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): or _update_func_cell_for__class__(member.fdel, cls, newcls)): break + # Get new annotations to remove references to the original class + # in forward references + newcls_ann = annotationlib.get_annotations( + newcls, format=annotationlib.Format.FORWARDREF) + + # Fix references in dataclass Fields + for f in getattr(newcls, _FIELDS).values(): + try: + ann = newcls_ann[f.name] + except KeyError: + pass + else: + f.type = ann + + # Fix the class reference in the __annotate__ method + init_annotate = newcls.__init__.__annotate__ + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) + return newcls diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 6bf5e5b3e55..513dd78c438 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -2471,6 +2471,135 @@ def __init__(self, a): self.assertEqual(D(5).a, 10) +class TestInitAnnotate(unittest.TestCase): + # Tests for the generated __annotate__ function for __init__ + # See: https://github.com/python/cpython/issues/137530 + + def test_annotate_function(self): + # No forward references + @dataclass + class A: + a: int + + value_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'a': int, 'return': None}) + self.assertEqual(forwardref_annos, {'a': int, 'return': None}) + self.assertEqual(string_annos, {'a': 'int', 'return': 'None'}) + + self.assertTrue(getattr(A.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_annotate_function_forwardref(self): + # With forward references + @dataclass + class B: + b: undefined + + # VALUE annotations should raise while unresolvable + with self.assertRaises(NameError): + _ = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(forwardref_annos, {'b': support.EqualToForwardRef('undefined', owner=B, is_class=True), 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + # Now VALUE and FORWARDREF should resolve, STRING should be unchanged + undefined = int + + value_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'b': int, 'return': None}) + self.assertEqual(forwardref_annos, {'b': int, 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + def test_annotate_function_init_false(self): + # Check `init=False` attributes don't get into the annotations of the __init__ function + @dataclass + class C: + c: str = field(init=False) + + self.assertEqual(annotationlib.get_annotations(C.__init__), {'return': None}) + + def test_annotate_function_contains_forwardref(self): + # Check string annotations on objects containing a ForwardRef + @dataclass + class D: + d: list[undefined] + + with self.assertRaises(NameError): + annotationlib.get_annotations(D.__init__) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[support.EqualToForwardRef("undefined", is_class=True, owner=D)], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + # Now test when it is defined + undefined = str + + # VALUE should now resolve + self.assertEqual( + annotationlib.get_annotations(D.__init__), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + def test_annotate_function_not_replaced(self): + # Check that __annotate__ is not replaced on non-generated __init__ functions + @dataclass(slots=True) + class E: + x: str + def __init__(self, x: int) -> None: + self.x = x + + self.assertEqual( + annotationlib.get_annotations(E.__init__), {"x": int, "return": None} + ) + + self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_init_false_forwardref(self): + # Test forward references in fields not required for __init__ annotations. + + # At the moment this raises a NameError for VALUE annotations even though the + # undefined annotation is not required for the __init__ annotations. + # Ideally this will be fixed but currently there is no good way to resolve this + + @dataclass + class F: + not_in_init: list[undefined] = field(init=False, default=None) + in_init: int + + annos = annotationlib.get_annotations(F.__init__, format=annotationlib.Format.FORWARDREF) + self.assertEqual( + annos, + {"in_init": int, "return": None}, + ) + + with self.assertRaises(NameError): + annos = annotationlib.get_annotations(F.__init__) # NameError on not_in_init + + class TestRepr(unittest.TestCase): def test_repr(self): @dataclass @@ -3831,7 +3960,15 @@ def method(self) -> int: return SlotsTest - for make in (make_simple, make_with_annotations, make_with_annotations_and_method): + def make_with_forwardref(): + @dataclass(slots=True) + class SlotsTest: + x: undefined + y: list[undefined] + + return SlotsTest + + for make in (make_simple, make_with_annotations, make_with_annotations_and_method, make_with_forwardref): with self.subTest(make=make): C = make() support.gc_collect() diff --git a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst new file mode 100644 index 00000000000..4ff55b41dea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst @@ -0,0 +1 @@ +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by replacing the annotations that were in-line in the generated source code with ``__annotate__`` functions attached to the methods. From 06b62282c79dd69293a3eefb4c55f5acc6312cb2 Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:15:22 +1030 Subject: [PATCH 104/417] gh-141174: Improve `annotationlib.get_annotations()` test coverage (#141286) * Test `get_annotations(format=Format.VALUE)` for stringized annotations on custom objects * Test `get_annotations(format=Format.VALUE)` for stringized annotations on wrapped partial functions * Update test_stringized_annotations_with_star_unpack() to actually test stringized annotations * Test __annotate__ returning a non-dict * Test passing globals and locals to stringized `get_annotations()` --- Lib/test/test_annotationlib.py | 70 +++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index fd5d43b09b9..f1d32ab50cf 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -9,6 +9,7 @@ import pickle from string.templatelib import Template, Interpolation import typing +import sys import unittest from annotationlib import ( Format, @@ -755,6 +756,8 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): @@ -788,7 +791,7 @@ def test_stringized_annotations_in_empty_module(self): self.assertEqual(get_annotations(isa2, eval_str=False), {}) def test_stringized_annotations_with_star_unpack(self): - def f(*args: *tuple[int, ...]): ... + def f(*args: "*tuple[int, ...]"): ... self.assertEqual(get_annotations(f, eval_str=True), {'args': (*tuple[int, ...],)[0]}) @@ -811,6 +814,44 @@ def test_stringized_annotations_on_wrapper(self): {"a": "int", "b": "str", "return": "MyClass"}, ) + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + def test_stringized_annotations_on_class(self): isa = inspect_stringized_annotations # test that local namespace lookups work @@ -823,6 +864,16 @@ def test_stringized_annotations_on_class(self): {"x": int}, ) + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + def test_stringized_annotation_permutations(self): def define_class(name, has_future, has_annos, base_text, extra_names=None): lines = [] @@ -990,6 +1041,23 @@ def __annotate__(self): {"x": "int"}, ) + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + def test_no_annotations(self): class CustomClass: pass From 68266c1f01e5791558cb088dfb0e26ecd577295e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Nov 2025 15:50:51 +0100 Subject: [PATCH 105/417] gh-141341: Rename COMPILER macro to _Py_COMPILER on Windows (#141342) --- ...-11-10-11-26-26.gh-issue-141341.OsO6-y.rst | 2 ++ PC/pyconfig.h | 28 +++++++++---------- Python/getcompiler.c | 4 +++ 3 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst diff --git a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst new file mode 100644 index 00000000000..460923b4d62 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst @@ -0,0 +1,2 @@ +On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name +conflicts. Patch by Victor Stinner. diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 0e8379387cd..a126fca6f5a 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -118,7 +118,7 @@ WIN32 is still required for the locale module. /* Microsoft C defines _MSC_VER, as does clang-cl.exe */ #ifdef _MSC_VER -/* We want COMPILER to expand to a string containing _MSC_VER's *value*. +/* We want _Py_COMPILER to expand to a string containing _MSC_VER's *value*. * This is horridly tricky, because the stringization operator only works * on macro arguments, and doesn't evaluate macros passed *as* arguments. */ @@ -148,7 +148,7 @@ WIN32 is still required for the locale module. #define MS_WIN64 #endif -/* set the COMPILER and support tier +/* set the _Py_COMPILER and support tier * * win_amd64 MSVC (x86_64-pc-windows-msvc): 1 * win32 MSVC (i686-pc-windows-msvc): 1 @@ -158,22 +158,22 @@ WIN32 is still required for the locale module. #ifdef MS_WIN64 #if defined(_M_X64) || defined(_M_AMD64) #if defined(__clang__) -#define COMPILER ("[Clang " __clang_version__ "] 64 bit (AMD64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[Clang " __clang_version__ "] 64 bit (AMD64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #elif defined(__INTEL_COMPILER) -#define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") #define PY_SUPPORT_TIER 1 #endif /* __clang__ */ #define PYD_PLATFORM_TAG "win_amd64" #elif defined(_M_ARM64) -#define COMPILER _Py_PASTE_VERSION("64 bit (ARM64)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (ARM64)") #define PY_SUPPORT_TIER 3 #define PYD_PLATFORM_TAG "win_arm64" #else -#define COMPILER _Py_PASTE_VERSION("64 bit (Unknown)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (Unknown)") #define PY_SUPPORT_TIER 0 #endif #endif /* MS_WIN64 */ @@ -220,22 +220,22 @@ typedef _W64 int Py_ssize_t; #if defined(MS_WIN32) && !defined(MS_WIN64) #if defined(_M_IX86) #if defined(__clang__) -#define COMPILER ("[Clang " __clang_version__ "] 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[Clang " __clang_version__ "] 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #elif defined(__INTEL_COMPILER) -#define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("32 bit (Intel)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (Intel)") #define PY_SUPPORT_TIER 1 #endif /* __clang__ */ #define PYD_PLATFORM_TAG "win32" #elif defined(_M_ARM) -#define COMPILER _Py_PASTE_VERSION("32 bit (ARM)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (ARM)") #define PYD_PLATFORM_TAG "win_arm32" #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("32 bit (Unknown)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (Unknown)") #define PY_SUPPORT_TIER 0 #endif #endif /* MS_WIN32 && !MS_WIN64 */ @@ -273,7 +273,7 @@ typedef int pid_t; #warning "Please use an up-to-date version of gcc! (>2.91 recommended)" #endif -#define COMPILER "[gcc]" +#define _Py_COMPILER "[gcc]" #define PY_LONG_LONG long long #define PY_LLONG_MIN LLONG_MIN #define PY_LLONG_MAX LLONG_MAX @@ -286,7 +286,7 @@ typedef int pid_t; /* XXX These defines are likely incomplete, but should be easy to fix. They should be complete enough to build extension modules. */ -#define COMPILER "[lcc-win32]" +#define _Py_COMPILER "[lcc-win32]" typedef int pid_t; /* __declspec() is supported here too - do nothing to get the defaults */ diff --git a/Python/getcompiler.c b/Python/getcompiler.c index a5d26239e87..cc56ad8c895 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -3,6 +3,10 @@ #include "Python.h" +#ifdef _Py_COMPILER +# define COMPILER _Py_COMPILER +#endif + #ifndef COMPILER // Note the __clang__ conditional has to come before the __GNUC__ one because From 19b573025e0aa569e7a34081116280133e33979a Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:23:40 +1030 Subject: [PATCH 106/417] gh-141174: Improve `ForwardRef` test coverage (#141175) * Test unsupported format in ForwardRef.evaluate() * Test dict cell closure with multiple variables * Test all options in ForwardRef repr * Test ForwardRef being a final class --- Lib/test/test_annotationlib.py | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index f1d32ab50cf..d196801eede 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -74,6 +74,30 @@ def inner(arg: x): anno = get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + def test_function(self): def f(x: int, y: doesntexist): pass @@ -1756,6 +1780,14 @@ def test_forward_repr(self): repr(List[ForwardRef("int", module="mod")]), "typing.List[ForwardRef('int', module='mod')]", ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) def test_forward_recursion_actually(self): def namespace1(): @@ -1861,6 +1893,19 @@ def test_evaluate_forwardref_format(self): support.EqualToForwardRef('"a" + 1'), ) + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + def test_evaluate_with_type_params(self): class Gen[T]: alias = int @@ -1994,6 +2039,11 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + class TestAnnotationLib(unittest.TestCase): def test__all__(self): From 1110e8f6a4a767f6d09b121017442528733b380b Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:24:50 +1030 Subject: [PATCH 107/417] gh-141174: Improve `annotationlib.call_annotate_function()` test coverage (#141176) * Test passing unsupported Format values to call_annotate_function() * Test call_evaluate_function with fake globals that raise errors * Fix typo and comparison in test_fake_global_evaluation --- Lib/test/test_annotationlib.py | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index d196801eede..0ae598b6839 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1339,6 +1339,32 @@ def evaluate(format, exc=NotImplementedError): "undefined", ) + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + class TestCallAnnotateFunction(unittest.TestCase): # Tests for user defined annotate functions. @@ -1480,6 +1506,23 @@ def annotate(format, /): with self.assertRaises(NotImplementedError): annotationlib.call_annotate_function(annotate, Format.STRING) + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + def test_error_from_value_raised(self): # Test that the error from format.VALUE is raised # if all formats fail From 59b793b0dd76d37229fe6d379cd5fe76023d15f1 Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Mon, 10 Nov 2025 22:55:15 +0800 Subject: [PATCH 108/417] gh-141343: Fix swapped words in `sorted` doc (GH-141348) --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 61799e303a1..e9879397555 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1859,7 +1859,7 @@ are always available. They are listed here in alphabetical order. the same data with other ordering tools such as :func:`max` that rely on a different underlying method. Implementing all six comparisons also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. + the reflected :meth:`~object.__gt__` method. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. From 55ea13231313a2133e6f5a6112409d349081f273 Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Tue, 11 Nov 2025 01:26:45 +1030 Subject: [PATCH 109/417] gh-141174: Improve `annotationlib._Stringifier` test coverage (#141220) * Test `_Stringifier.__convert_to_ast()` for containers * Test partial evaluation of `ForwardRef`s in `_Stringifier` --- Lib/test/test_annotationlib.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 0ae598b6839..9f3275d5071 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -119,6 +119,10 @@ def f( alpha: some | obj, beta: +some, gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj, module}, + zeta: some | [obj], + eta: some | (), ): pass @@ -147,6 +151,69 @@ def f( self.assertIsInstance(gamma_anno, ForwardRef) self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj, module}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + def test_partially_nonexistent_union(self): # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs class UnionForwardrefs: From 88953d5debf08dfaa1cdb314d62262f770addf5b Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 10 Nov 2025 18:36:01 +0300 Subject: [PATCH 110/417] gh-141004: Deprecate Py_MATH_El and Py_MATH_PIl macros (#141035) Co-authored-by: Victor Stinner --- Doc/c-api/float.rst | 14 ++++++++++++++ Doc/deprecations/c-api-pending-removal-in-3.20.rst | 2 ++ Doc/whatsnew/3.15.rst | 4 ++++ Include/pymath.h | 2 ++ .../2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst | 1 + 5 files changed, 23 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index edee498a0b8..9e703a46445 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,20 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + .. deprecated-removed:: 3.15 3.20 + + .. c:macro:: Py_RETURN_NAN Return :data:`math.nan` from a function. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst index 82f975d6ed4..18623b19a2a 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.20.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -5,3 +5,5 @@ Pending removal in Python 3.20 Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` to convert a Python complex number to/from the C :c:type:`Py_complex` representation. + +* Macros :c:macro:`!Py_MATH_PIl` and :c:macro:`!Py_MATH_El`. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e0b0471567c..1ba394a1967 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1076,6 +1076,10 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated + since 3.15 and will be removed in 3.20. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + .. Add C API deprecations above alphabetically, not here at the end. diff --git a/Include/pymath.h b/Include/pymath.h index e2919c7b527..0f9f0f3b299 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -7,6 +7,7 @@ /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ +// Deprecated since Python 3.15. #ifndef Py_MATH_PIl #define Py_MATH_PIl 3.1415926535897932384626433832795029L #endif @@ -14,6 +15,7 @@ #define Py_MATH_PI 3.14159265358979323846 #endif +// Deprecated since Python 3.15. #ifndef Py_MATH_El #define Py_MATH_El 2.7182818284590452353602874713526625L #endif diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst new file mode 100644 index 00000000000..5f3ccd62016 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst @@ -0,0 +1 @@ +:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. From f835552946e29ec20144c359b8822f9e421d4d64 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Mon, 10 Nov 2025 21:19:13 +0500 Subject: [PATCH 111/417] GH-141212: Fix possible memory leak in gc_mark_span_push (gh-141213) --- Python/gc_free_threading.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f39793c3eeb..b183062eff7 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -675,10 +675,11 @@ gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end) else { ss->capacity *= 2; } - ss->stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); - if (ss->stack == NULL) { + gc_span_t *new_stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); + if (new_stack == NULL) { return -1; } + ss->stack = new_stack; } assert(end > start); ss->stack[ss->size].start = start; From ed0a5fd8cacb1964111d03ff37627f6bea5e6026 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:46:41 +0000 Subject: [PATCH 112/417] gh-141004: Document `PyType_FastSubclass` (GH-141313) Co-authored-by: Peter Bierma --- Doc/c-api/type.rst | 12 ++++++++++++ Doc/c-api/typeobj.rst | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 5bdbff4e0ad..479ede70b01 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -133,6 +133,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 9d23aea5734..34d19acdf17 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1351,8 +1351,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` From 86513f6c2ebdd1fb692c39b84786ea41d88c84fd Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 10 Nov 2025 16:35:47 -0500 Subject: [PATCH 113/417] gh-141004: Document missing frame APIs (GH-141189) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/frame.rst | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1a52e146a69..fb17cf7f1da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -161,6 +167,57 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. + +Legacy Local Variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + + Internal Frames ^^^^^^^^^^^^^^^ From 46b58e1bb9e1e17d855588935f5a259be960a3a1 Mon Sep 17 00:00:00 2001 From: Louis Date: Tue, 11 Nov 2025 05:50:30 +0100 Subject: [PATCH 114/417] gh-140578: Doc: Remove sencence implying that concurrent.futures.ThreadPoolExecutor does not exist (#140689) * Doc: Remove sencence implying that concurrent.futures.ThreadPoolExecutor does not exist Closes #140578 * Add NEWS.d entry for gh-140578 --------- Co-authored-by: Louis Paternault --- Doc/library/multiprocessing.rst | 7 +++++-- .../2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index d18ada3511d..714207cb0ae 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -22,8 +22,7 @@ to this, the :mod:`multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the +The :mod:`multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -44,6 +43,10 @@ will print to standard output :: [1, 4, 9] +The :mod:`multiprocessing` module also introduces APIs which do not have +analogs in the :mod:`threading` module, like the ability to :meth:`terminate +`, :meth:`interrupt ` or :meth:`kill +` a running process. .. seealso:: diff --git a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst new file mode 100644 index 00000000000..702d38d4d24 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst @@ -0,0 +1,3 @@ +Remove outdated sencence in the documentation for :mod:`multiprocessing`, +that implied that :class:`concurrent.futures.ThreadPoolExecutor` did not +exist. From 9cb8c52d5e9a83efe4fa3878db06befd9df52f54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Nov 2025 05:59:16 +0100 Subject: [PATCH 115/417] gh-140485: Catch ChildProcessError in multiprocessing resource tracker (GH-141132) --- Lib/multiprocessing/resource_tracker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index c53092f6e34..38fcaed48fa 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -111,7 +111,12 @@ def _stop_locked( close(self._fd) self._fd = None - _, status = waitpid(self._pid, 0) + try: + _, status = waitpid(self._pid, 0) + except ChildProcessError: + self._pid = None + self._exitcode = None + return self._pid = None From 92741c59f89e114474bdb2cb539107ef6bae0b9c Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya <141550576+XChaitanyaX@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:32:46 +0530 Subject: [PATCH 116/417] gh-140379: add hyperlinks to list and set (GH-140399) add hyperlinks to list and set --- Doc/tutorial/datastructures.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 1332c53f396..7e02e74177c 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -12,9 +12,8 @@ and adds some new things as well. More on Lists ============= -The list data type has some more methods. Here are all of the methods of list -objects: - +The :ref:`list ` data type has some more methods. Here are all +of the methods of list objects: .. method:: list.append(x) :noindex: @@ -445,10 +444,11 @@ packing and sequence unpacking. Sets ==== -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. +Python also includes a data type for :ref:`sets `. A set is +an unordered collection with no duplicate elements. Basic uses include +membership testing and eliminating duplicate entries. Set objects also +support mathematical operations like union, intersection, difference, and +symmetric difference. Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an From 8435a2278f964f48d36edbc5092be5ebecfcb120 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Nov 2025 09:21:24 +0100 Subject: [PATCH 117/417] gh-141376: Fix exported symbols (GH-141377) * gh-141376: Fix exported symbols * _io module: add "_Py_" prefix to "spec" variables. For example, rename bufferedrandom_spec to _Py_bufferedrandom_spec. * typevarobject.c: add "static" to "spec" and "slots" variables. * import.c: add "static" to "pkgcontext" variable. * No longer export textiowrapper_slots --- Modules/_io/_iomodule.c | 30 +++++++++++++++--------------- Modules/_io/_iomodule.h | 30 +++++++++++++++--------------- Modules/_io/bufferedio.c | 10 +++++----- Modules/_io/bytesio.c | 4 ++-- Modules/_io/fileio.c | 2 +- Modules/_io/iobase.c | 4 ++-- Modules/_io/stringio.c | 2 +- Modules/_io/textio.c | 8 ++++---- Modules/_io/winconsoleio.c | 2 +- Objects/typevarobject.c | 16 ++++++++-------- Python/import.c | 2 +- 11 files changed, 55 insertions(+), 55 deletions(-) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 27483494559..433d68d515c 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -681,40 +681,40 @@ iomodule_exec(PyObject *m) } // Base classes - ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); - ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); - ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, NULL); + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &_Py_nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &_Py_bytesiobuf_spec, NULL); + ADD_TYPE(m, state->PyIOBase_Type, &_Py_iobase_spec, NULL); // PyIOBase_Type subclasses - ADD_TYPE(m, state->PyTextIOBase_Type, &textiobase_spec, + ADD_TYPE(m, state->PyTextIOBase_Type, &_Py_textiobase_spec, state->PyIOBase_Type); - ADD_TYPE(m, state->PyBufferedIOBase_Type, &bufferediobase_spec, + ADD_TYPE(m, state->PyBufferedIOBase_Type, &_Py_bufferediobase_spec, state->PyIOBase_Type); - ADD_TYPE(m, state->PyRawIOBase_Type, &rawiobase_spec, + ADD_TYPE(m, state->PyRawIOBase_Type, &_Py_rawiobase_spec, state->PyIOBase_Type); // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, + ADD_TYPE(m, state->PyBytesIO_Type, &_Py_bytesio_spec, state->PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedWriter_Type, &_Py_bufferedwriter_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, + ADD_TYPE(m, state->PyBufferedReader_Type, &_Py_bufferedreader_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, + ADD_TYPE(m, state->PyBufferedRWPair_Type, &_Py_bufferedrwpair_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, + ADD_TYPE(m, state->PyBufferedRandom_Type, &_Py_bufferedrandom_spec, state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); + ADD_TYPE(m, state->PyFileIO_Type, &_Py_fileio_spec, state->PyRawIOBase_Type); #ifdef HAVE_WINDOWS_CONSOLE_IO - ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, + ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &_Py_winconsoleio_spec, state->PyRawIOBase_Type); #endif // PyTextIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); - ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, + ADD_TYPE(m, state->PyStringIO_Type, &_Py_stringio_spec, state->PyTextIOBase_Type); + ADD_TYPE(m, state->PyTextIOWrapper_Type, &_Py_textiowrapper_spec, state->PyTextIOBase_Type); #undef ADD_TYPE diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 18cf20edf26..4ae487c8e2a 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -9,23 +9,23 @@ #include "structmember.h" /* Type specs */ -extern PyType_Spec bufferediobase_spec; -extern PyType_Spec bufferedrandom_spec; -extern PyType_Spec bufferedreader_spec; -extern PyType_Spec bufferedrwpair_spec; -extern PyType_Spec bufferedwriter_spec; -extern PyType_Spec bytesio_spec; -extern PyType_Spec bytesiobuf_spec; -extern PyType_Spec fileio_spec; -extern PyType_Spec iobase_spec; -extern PyType_Spec nldecoder_spec; -extern PyType_Spec rawiobase_spec; -extern PyType_Spec stringio_spec; -extern PyType_Spec textiobase_spec; -extern PyType_Spec textiowrapper_spec; +extern PyType_Spec _Py_bufferediobase_spec; +extern PyType_Spec _Py_bufferedrandom_spec; +extern PyType_Spec _Py_bufferedreader_spec; +extern PyType_Spec _Py_bufferedrwpair_spec; +extern PyType_Spec _Py_bufferedwriter_spec; +extern PyType_Spec _Py_bytesio_spec; +extern PyType_Spec _Py_bytesiobuf_spec; +extern PyType_Spec _Py_fileio_spec; +extern PyType_Spec _Py_iobase_spec; +extern PyType_Spec _Py_nldecoder_spec; +extern PyType_Spec _Py_rawiobase_spec; +extern PyType_Spec _Py_stringio_spec; +extern PyType_Spec _Py_textiobase_spec; +extern PyType_Spec _Py_textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO -extern PyType_Spec winconsoleio_spec; +extern PyType_Spec _Py_winconsoleio_spec; #endif /* These functions are used as METH_NOARGS methods, are normally called diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 0b4bc4c6b8a..4602f2b42a6 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2537,7 +2537,7 @@ static PyType_Slot bufferediobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec bufferediobase_spec = { +PyType_Spec _Py_bufferediobase_spec = { .name = "_io._BufferedIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), @@ -2600,7 +2600,7 @@ static PyType_Slot bufferedreader_slots[] = { {0, NULL}, }; -PyType_Spec bufferedreader_spec = { +PyType_Spec _Py_bufferedreader_spec = { .name = "_io.BufferedReader", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2658,7 +2658,7 @@ static PyType_Slot bufferedwriter_slots[] = { {0, NULL}, }; -PyType_Spec bufferedwriter_spec = { +PyType_Spec _Py_bufferedwriter_spec = { .name = "_io.BufferedWriter", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2708,7 +2708,7 @@ static PyType_Slot bufferedrwpair_slots[] = { {0, NULL}, }; -PyType_Spec bufferedrwpair_spec = { +PyType_Spec _Py_bufferedrwpair_spec = { .name = "_io.BufferedRWPair", .basicsize = sizeof(rwpair), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2776,7 +2776,7 @@ static PyType_Slot bufferedrandom_slots[] = { {0, NULL}, }; -PyType_Spec bufferedrandom_spec = { +PyType_Spec _Py_bufferedrandom_spec = { .name = "_io.BufferedRandom", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 30d61f9d68e..d6bfb93177c 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1156,7 +1156,7 @@ static PyType_Slot bytesio_slots[] = { {0, NULL}, }; -PyType_Spec bytesio_spec = { +PyType_Spec _Py_bytesio_spec = { .name = "_io.BytesIO", .basicsize = sizeof(bytesio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -1246,7 +1246,7 @@ static PyType_Slot bytesiobuf_slots[] = { {0, NULL}, }; -PyType_Spec bytesiobuf_spec = { +PyType_Spec _Py_bytesiobuf_spec = { .name = "_io._BytesIOBuffer", .basicsize = sizeof(bytesiobuf), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b84c1bd3e22..2544ff4ea91 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -1329,7 +1329,7 @@ static PyType_Slot fileio_slots[] = { {0, NULL}, }; -PyType_Spec fileio_spec = { +PyType_Spec _Py_fileio_spec = { .name = "_io.FileIO", .basicsize = sizeof(fileio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index e304fc8bee2..f1c2fe17801 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -885,7 +885,7 @@ static PyType_Slot iobase_slots[] = { {0, NULL}, }; -PyType_Spec iobase_spec = { +PyType_Spec _Py_iobase_spec = { .name = "_io._IOBase", .basicsize = sizeof(iobase), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -1046,7 +1046,7 @@ static PyType_Slot rawiobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec rawiobase_spec = { +PyType_Spec _Py_rawiobase_spec = { .name = "_io._RawIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 20b7cfc0088..781ca4327f9 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1094,7 +1094,7 @@ static PyType_Slot stringio_slots[] = { {0, NULL}, }; -PyType_Spec stringio_spec = { +PyType_Spec _Py_stringio_spec = { .name = "_io.StringIO", .basicsize = sizeof(stringio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index c462bd2ac57..84b7d9df400 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -208,7 +208,7 @@ static PyType_Slot textiobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec textiobase_spec = { +PyType_Spec _Py_textiobase_spec = { .name = "_io._TextIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), @@ -3352,7 +3352,7 @@ static PyType_Slot nldecoder_slots[] = { {0, NULL}, }; -PyType_Spec nldecoder_spec = { +PyType_Spec _Py_nldecoder_spec = { .name = "_io.IncrementalNewlineDecoder", .basicsize = sizeof(nldecoder_object), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -3404,7 +3404,7 @@ static PyGetSetDef textiowrapper_getset[] = { {NULL} }; -PyType_Slot textiowrapper_slots[] = { +static PyType_Slot textiowrapper_slots[] = { {Py_tp_dealloc, textiowrapper_dealloc}, {Py_tp_repr, textiowrapper_repr}, {Py_tp_doc, (void *)_io_TextIOWrapper___init____doc__}, @@ -3418,7 +3418,7 @@ PyType_Slot textiowrapper_slots[] = { {0, NULL}, }; -PyType_Spec textiowrapper_spec = { +PyType_Spec _Py_textiowrapper_spec = { .name = "_io.TextIOWrapper", .basicsize = sizeof(textio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 950b7fe241c..677d7e85d4e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1253,7 +1253,7 @@ static PyType_Slot winconsoleio_slots[] = { {0, NULL}, }; -PyType_Spec winconsoleio_spec = { +PyType_Spec _Py_winconsoleio_spec = { .name = "_io._WindowsConsoleIO", .basicsize = sizeof(winconsoleio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 75a69d4bc3e..8e43962c7e3 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -251,7 +251,7 @@ static PyType_Slot constevaluator_slots[] = { {0, NULL}, }; -PyType_Spec constevaluator_spec = { +static PyType_Spec constevaluator_spec = { .name = "_typing._ConstEvaluator", .basicsize = sizeof(constevaluatorobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -930,7 +930,7 @@ static PyType_Slot typevar_slots[] = { {0, NULL}, }; -PyType_Spec typevar_spec = { +static PyType_Spec typevar_spec = { .name = "typing.TypeVar", .basicsize = sizeof(typevarobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1078,7 +1078,7 @@ static PyType_Slot paramspecargs_slots[] = { {0, NULL}, }; -PyType_Spec paramspecargs_spec = { +static PyType_Spec paramspecargs_spec = { .name = "typing.ParamSpecArgs", .basicsize = sizeof(paramspecattrobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1158,7 +1158,7 @@ static PyType_Slot paramspeckwargs_slots[] = { {0, NULL}, }; -PyType_Spec paramspeckwargs_spec = { +static PyType_Spec paramspeckwargs_spec = { .name = "typing.ParamSpecKwargs", .basicsize = sizeof(paramspecattrobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1509,7 +1509,7 @@ static PyType_Slot paramspec_slots[] = { {0, 0}, }; -PyType_Spec paramspec_spec = { +static PyType_Spec paramspec_spec = { .name = "typing.ParamSpec", .basicsize = sizeof(paramspecobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1789,7 +1789,7 @@ Note that only TypeVarTuples defined in the global scope can be\n\ pickled.\n\ "); -PyType_Slot typevartuple_slots[] = { +static PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, @@ -1805,7 +1805,7 @@ PyType_Slot typevartuple_slots[] = { {0, 0}, }; -PyType_Spec typevartuple_spec = { +static PyType_Spec typevartuple_spec = { .name = "typing.TypeVarTuple", .basicsize = sizeof(typevartupleobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_DICT @@ -2347,7 +2347,7 @@ static PyType_Slot generic_slots[] = { {0, NULL}, }; -PyType_Spec generic_spec = { +static PyType_Spec generic_spec = { .name = "typing.Generic", .basicsize = sizeof(PyObject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, diff --git a/Python/import.c b/Python/import.c index 6cf4a061ca6..2afa7c15e6a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -804,7 +804,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) substitute this (if the name actually matches). */ -_Py_thread_local const char *pkgcontext = NULL; +static _Py_thread_local const char *pkgcontext = NULL; # undef PKGCONTEXT # define PKGCONTEXT pkgcontext From d69447445cbacf7537bf59c5c683a3b17060312d Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 11 Nov 2025 13:13:59 +0300 Subject: [PATCH 118/417] gh-141004: document Py_INFINITY and Py_NAN macros (#141145) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/float.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 9e703a46445..eae4792af7d 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,24 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands a to constant expression of type :c:expr:`double`, that + represents the positive infinity. + + On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_NAN + + This macro expands a to constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + .. c:macro:: Py_MATH_El High precision (long double) definition of :data:`~math.e` constant. From 799326b0a93ae6375f153d5a6607e7dc5e0690b2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 11 Nov 2025 13:52:13 +0100 Subject: [PATCH 119/417] gh-141169: Re-raise exception from findfuncptr (GH-141349) --- Include/internal/pycore_importdl.h | 32 ++++++++++++++++++++++++++---- Python/importdl.c | 24 +++------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 12a32a5f70e..f60c5510d20 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,6 +14,34 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; +#ifdef HAVE_DYNAMIC_LOADING +/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is + supported on this platform. configure will then compile and link in one + of the dynload_*.c files, as appropriate. We will call a function in + those modules to get a function pointer to the module's init function. + + The function should return: + - The function pointer on success + - NULL with exception set if the library cannot be loaded + - NULL *without* an extension set if the library could be loaded but the + function cannot be found in it. +*/ +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; +extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, + const char *shortname, + PyObject *pathname, + FILE *fp); +#else +typedef void (*dl_funcptr)(void); +extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, + const char *shortname, + const char *pathname, FILE *fp); +#endif + +#endif /* HAVE_DYNAMIC_LOADING */ + typedef enum ext_module_kind { _Py_ext_module_kind_UNKNOWN = 0, @@ -112,8 +140,6 @@ extern int _PyImport_RunModInitFunc( #define MAXSUFFIXSIZE 12 #ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; #ifdef Py_DEBUG # define PYD_DEBUG_SUFFIX "_d" @@ -136,8 +162,6 @@ typedef FARPROC dl_funcptr; #define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -#else -typedef void (*dl_funcptr)(void); #endif diff --git a/Python/importdl.c b/Python/importdl.c index 23a55c39677..61a9cdaf375 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -10,27 +10,6 @@ #include "pycore_runtime.h" // _Py_ID() -/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is - supported on this platform. configure will then compile and link in one - of the dynload_*.c files, as appropriate. We will call a function in - those modules to get a function pointer to the module's init function. -*/ -#ifdef HAVE_DYNAMIC_LOADING - -#ifdef MS_WINDOWS -extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, - const char *shortname, - PyObject *pathname, - FILE *fp); -#else -extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, - const char *shortname, - const char *pathname, FILE *fp); -#endif - -#endif /* HAVE_DYNAMIC_LOADING */ - - /***********************************/ /* module info to use when loading */ /***********************************/ @@ -414,6 +393,9 @@ _PyImport_GetModuleExportHooks( *modexport = (PyModExportFunction)exportfunc; return 2; } + if (PyErr_Occurred()) { + return -1; + } exportfunc = findfuncptr( info->hook_prefixes->init_prefix, From 7211a34fe1d9704935342af8c9b46725629f2d97 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 11 Nov 2025 20:02:32 +0530 Subject: [PATCH 120/417] gh-132657: optimize `PySet_Contains` for `frozenset` (#141183) --- Objects/setobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 213bd821d8a..2401176576e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2747,7 +2747,9 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - + if (PyFrozenSet_CheckExact(anyset)) { + return set_contains_key((PySetObject *)anyset, key); + } int rv; Py_BEGIN_CRITICAL_SECTION(anyset); rv = set_contains_key((PySetObject *)anyset, key); From d890aba748e5213585f9f906888999227dc3fa9c Mon Sep 17 00:00:00 2001 From: John Franey <1728528+johnfraney@users.noreply.github.com> Date: Tue, 11 Nov 2025 10:33:56 -0400 Subject: [PATCH 121/417] gh-140942: Add MIME type for .cjs extension (#140937) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.15.rst | 1 + Lib/mimetypes.py | 1 + .../2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1ba394a1967..ef18d36e4d4 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -451,6 +451,7 @@ math mimetypes --------- +* Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 48a9f430d45..d6896fc4042 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -486,6 +486,7 @@ def _default_mime_types(): '.wiz' : 'application/msword', '.nq' : 'application/n-quads', '.nt' : 'application/n-triples', + '.cjs' : 'application/node', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', '.dll' : 'application/octet-stream', diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst new file mode 100644 index 00000000000..20cfeca1e71 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst @@ -0,0 +1,2 @@ +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. From 759a048d4bea522fda2fe929be0fba1650c62b0e Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 12:22:16 -0500 Subject: [PATCH 122/417] gh-141004: Document `PyType_Unwatch` (GH-141414) --- Doc/c-api/type.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 479ede70b01..29ffeb7c483 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -116,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. From 713edbcebfdb5aa83e9bf376ebc40255ccacd235 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Wed, 12 Nov 2025 03:27:21 +0800 Subject: [PATCH 123/417] gh-141415: Remove unused variables and comment in `_pyrepl.windows_console.py` (#141416) --- Lib/_pyrepl/windows_console.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index c56dcd6d7dd..f9f5988af0b 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -249,22 +249,10 @@ def input_hook(self): def __write_changed_line( self, y: int, oldline: str, newline: str, px_coord: int ) -> None: - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) minlen = min(wlen(oldline), wlen(newline)) x_pos = 0 x_coord = 0 - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape # sequence @@ -358,7 +346,6 @@ def prepare(self) -> None: self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 self.__offset = 0 if self.__vt_support: From 0f09bda643d778fb20fb79fecdfd09f20f9d9717 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 03:27:56 +0800 Subject: [PATCH 124/417] gh-140193: Forward port test_exec_set_nomemory_hang from 3.13 (GH-140187) * chore: test_exec_set_nomemory_hang from 3.13 Signed-off-by: yihong0618 * fix: apply comments Signed-off-by: yihong0618 * Update Lib/test/test_exceptions.py Co-authored-by: Peter Bierma * Update Lib/test/test_exceptions.py Co-authored-by: Peter Bierma * fix: windows too long name 60 times is enough Signed-off-by: yihong0618 --------- Signed-off-by: yihong0618 Co-authored-by: Peter Bierma --- Lib/test/test_exceptions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 5262b58908a..6f212d2f91e 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1923,6 +1923,39 @@ def test_keyerror_context(self): exc2 = None + @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_exec_set_nomemory_hang(self): + import_module("_testcapi") + # gh-134163: A MemoryError inside code that was wrapped by a try/except + # block would lead to an infinite loop. + + # The frame_lasti needs to be greater than 257 to prevent + # PyLong_FromLong() from returning cached integers, which + # don't require a memory allocation. Prepend some dummy code + # to artificially increase the instruction index. + warmup_code = "a = list(range(0, 1))\n" * 60 + user_input = warmup_code + dedent(""" + try: + import _testcapi + _testcapi.set_nomemory(0) + b = list(range(1000, 2000)) + except Exception as e: + import traceback + traceback.print_exc() + """) + with SuppressCrashReport(): + with script_helper.spawn_python('-c', user_input) as p: + p.wait() + output = p.stdout.read() + + self.assertIn(p.returncode, (0, 1)) + self.assertGreater(len(output), 0) # At minimum, should not hang + self.assertIn(b"MemoryError", output) + + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): try: From c903d768322989e9f8ba79e38ee87e14c85c5430 Mon Sep 17 00:00:00 2001 From: Marco Barbosa Date: Tue, 11 Nov 2025 16:35:55 -0300 Subject: [PATCH 125/417] gh-139533: fix refs to code without proper markups on turtledemo doc (GH-139534) gh-139533: fix refs to code without proper markups on turtledemo documentation --- Doc/library/turtle.rst | 124 ++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 58b99e0d441..95a57c57e71 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2801,68 +2801,68 @@ The demo scripts are: .. tabularcolumns:: |l|L|L| -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| rosette | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ ++------------------------+------------------------------+--------------------------------------+ +| Name | Description | Features | ++========================+==============================+======================================+ +| ``bytedesign`` | complex classical | :func:`tracer`, :func:`delay`, | +| | turtle graphics pattern | :func:`update` | ++------------------------+------------------------------+--------------------------------------+ +| ``chaos`` | graphs Verhulst dynamics, | world coordinates | +| | shows that computer's | | +| | computations can generate | | +| | results sometimes against the| | +| | common sense expectations | | ++------------------------+------------------------------+--------------------------------------+ +| ``clock`` | analog clock showing time | turtles as clock's | +| | of your computer | hands, :func:`ontimer` | ++------------------------+------------------------------+--------------------------------------+ +| ``colormixer`` | experiment with r, g, b | :func:`ondrag` | ++------------------------+------------------------------+--------------------------------------+ +| ``forest`` | 3 breadth-first trees | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``fractalcurves`` | Hilbert & Koch curves | recursion | ++------------------------+------------------------------+--------------------------------------+ +| ``lindenmayer`` | ethnomathematics | L-System | +| | (indian kolams) | | ++------------------------+------------------------------+--------------------------------------+ +| ``minimal_hanoi`` | Towers of Hanoi | Rectangular Turtles | +| | | as Hanoi discs | +| | | (:func:`shape`, :func:`shapesize`) | ++------------------------+------------------------------+--------------------------------------+ +| ``nim`` | play the classical nim game | turtles as nimsticks, | +| | with three heaps of sticks | event driven (mouse, | +| | against the computer. | keyboard) | ++------------------------+------------------------------+--------------------------------------+ +| ``paint`` | super minimalistic | :func:`onclick` | +| | drawing program | | ++------------------------+------------------------------+--------------------------------------+ +| ``peace`` | elementary | turtle: appearance | +| | | and animation | ++------------------------+------------------------------+--------------------------------------+ +| ``penrose`` | aperiodic tiling with | :func:`stamp` | +| | kites and darts | | ++------------------------+------------------------------+--------------------------------------+ +| ``planet_and_moon`` | simulation of | compound shapes, | +| | gravitational system | :class:`Vec2D` | ++------------------------+------------------------------+--------------------------------------+ +| ``rosette`` | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++------------------------+------------------------------+--------------------------------------+ +| ``round_dance`` | dancing turtles rotating | compound shapes, :func:`clone` | +| | pairwise in opposite | :func:`shapesize`, :func:`tilt`, | +| | direction | :func:`get_shapepoly`, :func:`update`| ++------------------------+------------------------------+--------------------------------------+ +| ``sorting_animate`` | visual demonstration of | simple alignment, | +| | different sorting methods | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``tree`` | a (graphical) breadth | :func:`clone` | +| | first tree (using generators)| | ++------------------------+------------------------------+--------------------------------------+ +| ``two_canvases`` | simple design | turtles on two | +| | | canvases | ++------------------------+------------------------------+--------------------------------------+ +| ``yinyang`` | another elementary example | :func:`circle` | ++------------------------+------------------------------+--------------------------------------+ Have fun! From 336154f4b0dbcf1d9dbb461ae962d558ba60f452 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 11 Nov 2025 20:02:49 +0000 Subject: [PATCH 126/417] Add documentation for Python install manager's install_dir, global_dir and download_dir (GH-140223) --- Doc/using/windows.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 0b98cfb8d27..e6619b73bd2 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -457,6 +457,25 @@ customization. - Specify the default format used by the ``py list`` command. By default, ``table``. + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe``) + are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. From 2fb2b82161c6df57c4a247cb743816b79134e932 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Tue, 11 Nov 2025 23:16:46 +0300 Subject: [PATCH 127/417] gh-141367: Use actual SPECIALIZATION_THRESHOLD value in specialization related test (GH-141417) --- Lib/test/test_list.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 223f34fb696..642b54d3484 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -349,10 +349,12 @@ def test_deopt_from_append_list(self): # gh-132011: it used to crash, because # of `CALL_LIST_APPEND` specialization failure. code = textwrap.dedent(""" + import _testinternalcapi + l = [] def lappend(l, x, y): l.append((x, y)) - for x in range(3): + for x in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): lappend(l, None, None) try: lappend(list, None, None) From b5196fa15a6c5aaa90eafff06206f8e44a9da216 Mon Sep 17 00:00:00 2001 From: Aniket <148300120+Aniketsy@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:55:26 +0530 Subject: [PATCH 128/417] gh-137339: Clarify host and port parameter behavior in smtplib.SMTP{_SSL} initialization (#137340) This also documents the previously undocumented default_port parameter. Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/smtplib.rst | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5a3de52090..3ee8b82a188 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,10 +24,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + that support a full repertoire of SMTP and ESMTP operations. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + host and port automatically when the object is created; otherwise, :meth:`!connect` must + be called manually. + + If specified, *local_hostname* is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional @@ -62,6 +65,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, where ``data`` is the bytes about to be sent to the remote host. + .. attribute:: SMTP.default_port + + The default port used for SMTP connections (25). + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -80,15 +87,23 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`~SMTP.starttls` - is not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same + required from the beginning of the connection and using :meth:`SMTP.starttls` is + not appropriate. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must + be called manually. + + The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, can contain a :class:`~ssl.SSLContext` and allows configuring various aspects of the secure connection. Please read :ref:`ssl-security` for best practices. + .. attribute:: SMTP_SSL.default_port + + The default port used for SMTP-over-SSL connections (465). + .. versionchanged:: 3.3 *context* was added. @@ -259,6 +274,9 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + If port is not changed from its default value of 0, the value of the :attr:`default_port` + attribute is used. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect From 298e9074cdffb09d518e6aceea556e8f4a8a745d Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 11 Nov 2025 12:27:21 -0800 Subject: [PATCH 129/417] gh-140476: optimize `PySet_Add` for `frozenset` in free-threading (#140440) Avoids critical section in `PySet_Add` when adding items to newly created frozensets. Co-authored-by: Kumar Aditya --- ...-10-22-12-48-05.gh-issue-140476.F3-d1P.rst | 2 ++ Objects/setobject.c | 25 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst new file mode 100644 index 00000000000..a24033208c5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst @@ -0,0 +1,2 @@ +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +` build. diff --git a/Objects/setobject.c b/Objects/setobject.c index 2401176576e..85f4d7d4031 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2775,17 +2775,24 @@ PySet_Discard(PyObject *set, PyObject *key) int PySet_Add(PyObject *anyset, PyObject *key) { - if (!PySet_Check(anyset) && - (!PyFrozenSet_Check(anyset) || !_PyObject_IsUniquelyReferenced(anyset))) { - PyErr_BadInternalCall(); - return -1; + if (PySet_Check(anyset)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } - int rv; - Py_BEGIN_CRITICAL_SECTION(anyset); - rv = set_add_key((PySetObject *)anyset, key); - Py_END_CRITICAL_SECTION(); - return rv; + if (PyFrozenSet_Check(anyset) && _PyObject_IsUniquelyReferenced(anyset)) { + // We can only change frozensets if they are uniquely referenced. The + // API limits the usage of `PySet_Add` to "fill in the values of brand + // new frozensets before they are exposed to other code". In this case, + // this can be done without a lock. + return set_add_key((PySetObject *)anyset, key); + } + + PyErr_BadInternalCall(); + return -1; } int From 2befce86e699fdbb6610949b029bad56a0d0780f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 15:31:29 -0500 Subject: [PATCH 130/417] gh-141004: Document `PyFile_OpenCode` and `PyFile_OpenCodeObject` (GH-141413) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/file.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500..9d01254ddb2 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) From c13b59204af562bfb022eb8f6a5c03eb82659531 Mon Sep 17 00:00:00 2001 From: Alper Date: Tue, 11 Nov 2025 12:31:55 -0800 Subject: [PATCH 131/417] gh-116738: use `PyMutex` in `lzma` module (#140711) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_lzma.py | 56 +++++++++++++++++++++++ Modules/_lzmamodule.c | 46 ++++++------------- 2 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 Lib/test/test_free_threading/test_lzma.py diff --git a/Lib/test/test_free_threading/test_lzma.py b/Lib/test/test_free_threading/test_lzma.py new file mode 100644 index 00000000000..38d7e5db489 --- /dev/null +++ b/Lib/test/test_free_threading/test_lzma.py @@ -0,0 +1,56 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +lzma = import_helper.import_module("lzma") +from lzma import LZMACompressor, LZMADecompressor + +from test.test_lzma import INPUT + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestLZMA(unittest.TestCase): + def test_compressor(self): + lzc = LZMACompressor() + + # First compress() outputs LZMA header + header = lzc.compress(INPUT) + self.assertGreater(len(header), 0) + + def worker(): + # it should return empty bytes as it buffers data internally + data = lzc.compress(INPUT) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS - 1) + full_compressed = header + lzc.flush() + decompressed = lzma.decompress(full_compressed) + # The decompressed data should be INPUT repeated NTHREADS times + self.assertEqual(decompressed, INPUT * NTHREADS) + + def test_decompressor(self): + chunk_size = 128 + chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)] + input_data = b"".join(chunks) + compressed = lzma.compress(input_data) + + lzd = LZMADecompressor() + output = [] + + def worker(): + data = lzd.decompress(compressed, chunk_size) + self.assertEqual(len(data), chunk_size) + output.append(data) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(output), NTHREADS) + # Verify the expected chunks (order doesn't matter due to append race) + self.assertSetEqual(set(output), set(chunks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 6fc072f6d0a..58766233998 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -72,13 +72,6 @@ OutputBuffer_OnError(_BlocksOutputBuffer *buffer) } -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) typedef struct { PyTypeObject *lzma_compressor_type; @@ -111,7 +104,7 @@ typedef struct { lzma_allocator alloc; lzma_stream lzs; int flushed; - PyThread_type_lock lock; + PyMutex mutex; } Compressor; typedef struct { @@ -124,7 +117,7 @@ typedef struct { char needs_input; uint8_t *input_buffer; size_t input_buffer_size; - PyThread_type_lock lock; + PyMutex mutex; } Decompressor; #define Compressor_CAST(op) ((Compressor *)(op)) @@ -617,14 +610,14 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); } else { result = compress(self, data->buf, data->len, LZMA_RUN); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -644,14 +637,14 @@ _lzma_LZMACompressor_flush_impl(Compressor *self) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); } else { self->flushed = 1; result = compress(self, NULL, 0, LZMA_FINISH); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -820,12 +813,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->alloc.free = PyLzma_Free; self->lzs.allocator = &self->alloc; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->flushed = 0; switch (format) { @@ -867,10 +855,8 @@ static void Compressor_dealloc(PyObject *op) { Compressor *self = Compressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); lzma_end(&self->lzs); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -1146,12 +1132,12 @@ _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->eof) PyErr_SetString(PyExc_EOFError, "Already at end of stream"); else result = decompress(self, data->buf, data->len, max_length); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -1244,12 +1230,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, self->lzs.allocator = &self->alloc; self->lzs.next_in = NULL; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->check = LZMA_CHECK_UNKNOWN; self->needs_input = 1; @@ -1304,14 +1285,13 @@ static void Decompressor_dealloc(PyObject *op) { Decompressor *self = Decompressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); + if(self->input_buffer != NULL) PyMem_Free(self->input_buffer); lzma_end(&self->lzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); From 37e2762ee12c2d7fc465938d7161a9a0640bd71f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 11 Nov 2025 15:32:54 -0500 Subject: [PATCH 132/417] gh-141004: Document `PyBytes_Repr` and `PyBytes_DecodeEscape` (GH-141407) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/bytes.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 865a9e5d2bf..82c25573683 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -228,6 +228,42 @@ called with a non-bytes parameter. The function is :term:`soft deprecated`, use the :c:type:`PyBytesWriter` API instead. + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + .. _pybyteswriter: PyBytesWriter From af80fac42548719ede7241bfbab3c2c0775b4760 Mon Sep 17 00:00:00 2001 From: Mohsin Mehmood <55545648+mohsinm-dev@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:49:54 +0500 Subject: [PATCH 133/417] gh-141314: Fix TextIOWrapper.tell() assertion failure with standalone carriage return (GH-141331) The assertion was checking wrong variable (skip_back vs skip_bytes). --- Lib/test/test_io/test_textio.py | 19 +++++++++++++++++++ ...-11-10-01-47-18.gh-issue-141314.baaa28.rst | 1 + Modules/_io/textio.c | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index d8d0928b4ba..6331ed2b958 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -686,6 +686,25 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") diff --git a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst new file mode 100644 index 00000000000..37acaabfa3e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst @@ -0,0 +1 @@ +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files with standalone carriage return (``\r``) line endings. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 84b7d9df400..65da300abcf 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2845,7 +2845,7 @@ _io_TextIOWrapper_tell_impl(textio *self) current pos */ skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); skip_back = 1; - assert(skip_back <= PyBytes_GET_SIZE(next_input)); + assert(skip_bytes <= PyBytes_GET_SIZE(next_input)); input = PyBytes_AS_STRING(next_input); while (skip_bytes > 0) { /* Decode up to temptative start point */ From c744ccb2c92746bc7be6316ab478dbc13e176e97 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 11 Nov 2025 21:51:22 +0000 Subject: [PATCH 134/417] GH-139596: Cease caching config.cache & ccache in GH Actions (GH-139623) * Cease caching config.cache in GH Actions\ * Remove ccache action --- .github/workflows/build.yml | 52 --------------------------- .github/workflows/reusable-macos.yml | 5 --- .github/workflows/reusable-san.yml | 10 ------ .github/workflows/reusable-ubuntu.yml | 10 ------ .github/workflows/reusable-wasi.yml | 21 +---------- 5 files changed, 1 insertion(+), 97 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6aa99928278..a0f60c30ac8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,20 +109,10 @@ jobs: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -278,11 +268,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -304,10 +289,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython @@ -339,11 +320,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -370,10 +346,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | ./configure CFLAGS="-fdiagnostics-format=json" \ @@ -479,10 +451,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -493,11 +461,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -581,11 +544,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -611,11 +569,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -662,11 +615,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 3d310ae695b..d85c46b96f8 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -36,11 +36,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index e6ff02e4838..7fe96d1b238 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -34,11 +34,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh @@ -77,11 +72,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: >- ./configure diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7f8b9fdf5d6..7b93b5f51b0 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -64,11 +64,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +74,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 18feb564822..8f412288f53 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -42,11 +42,6 @@ jobs: mkdir "${WASI_SDK_PATH}" && \ curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \ tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: "Add ccache to PATH" run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" @@ -55,24 +50,10 @@ jobs: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi/__main__.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi/__main__.py') }}-${{ env.pythonLocation }} + run: python3 Tools/wasm/wasi.py make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi configure-host -- --config-cache From 7906f4d96a8fffbee9f4d4991019595878ad54e9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Nov 2025 00:01:25 +0200 Subject: [PATCH 135/417] gh-132686: Add parameters inherit_class_doc and fallback_to_class_doc for inspect.getdoc() (GH-132691) --- Doc/library/inspect.rst | 17 +++- Doc/whatsnew/3.15.rst | 8 ++ Lib/inspect.py | 34 +++++-- Lib/pydoc.py | 92 +------------------ Lib/test/test_inspect/test_inspect.py | 27 ++++++ ...-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst | 2 + 6 files changed, 79 insertions(+), 101 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 2b3b294ff33..aff53b78c4a 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -619,17 +619,26 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) Get the documentation string for an object, cleaned up with :func:`cleandoc`. - If the documentation string for an object is not provided and the object is - a class, a method, a property or a descriptor, retrieve the documentation - string from the inheritance hierarchy. + If the documentation string for an object is not provided: + + * if the object is a class and *inherit_class_doc* is true (by default), + retrieve the documentation string from the inheritance hierarchy; + * if the object is a method, a property or a descriptor, retrieve + the documentation string from the inheritance hierarchy; + * otherwise, if *fallback_to_class_doc* is true (by default), retrieve + the documentation string from the class of the object. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. + .. versionchanged:: next + Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + .. function:: getcomments(object) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ef18d36e4d4..ecab0d03e10 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -429,6 +429,14 @@ http.cookies (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) +inspect +------- + +* Add parameters *inherit_class_doc* and *fallback_to_class_doc* + for :func:`~inspect.getdoc`. + (Contributed by Serhiy Storchaka in :gh:`132686`.) + + locale ------ diff --git a/Lib/inspect.py b/Lib/inspect.py index bb22bab3040..bb17848b444 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -706,8 +706,8 @@ def _findclass(func): return None return cls -def _finddoc(obj): - if isclass(obj): +def _finddoc(obj, *, search_in_class=True): + if search_in_class and isclass(obj): for base in obj.__mro__: if base is not object: try: @@ -767,19 +767,37 @@ def _finddoc(obj): return doc return None -def getdoc(object): +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def getdoc(object, *, fallback_to_class_doc=True, inherit_class_doc=True): """Get the documentation string for an object. All tabs are expanded to spaces. To clean up docstrings that are indented to line up with blocks of code, any whitespace than can be uniformly removed from the second line onwards is removed.""" - try: - doc = object.__doc__ - except AttributeError: - return None + if fallback_to_class_doc: + try: + doc = object.__doc__ + except AttributeError: + return None + else: + doc = _getowndoc(object) if doc is None: try: - doc = _finddoc(object) + doc = _finddoc(object, search_in_class=inherit_class_doc) except (AttributeError, TypeError): return None if not isinstance(doc, str): diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 989fbd517d8..45ff5fca308 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -108,96 +108,10 @@ def pathdirs(): normdirs.append(normdir) return dirs -def _findclass(func): - cls = sys.modules.get(func.__module__) - if cls is None: - return None - for name in func.__qualname__.split('.')[:-1]: - cls = getattr(cls, name) - if not inspect.isclass(cls): - return None - return cls - -def _finddoc(obj): - if inspect.ismethod(obj): - name = obj.__func__.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - getattr(getattr(self, name, None), '__func__') is obj.__func__): - # classmethod - cls = self - else: - cls = self.__class__ - elif inspect.isfunction(obj): - name = obj.__name__ - cls = _findclass(obj) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.isbuiltin(obj): - name = obj.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - self.__qualname__ + '.' + name == obj.__qualname__): - # classmethod - cls = self - else: - cls = self.__class__ - # Should be tested before isdatadescriptor(). - elif isinstance(obj, property): - name = obj.__name__ - cls = _findclass(obj.fget) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): - name = obj.__name__ - cls = obj.__objclass__ - if getattr(cls, name) is not obj: - return None - if inspect.ismemberdescriptor(obj): - slots = getattr(cls, '__slots__', None) - if isinstance(slots, dict) and name in slots: - return slots[name] - else: - return None - for base in cls.__mro__: - try: - doc = _getowndoc(getattr(base, name)) - except AttributeError: - continue - if doc is not None: - return doc - return None - -def _getowndoc(obj): - """Get the documentation string for an object if it is not - inherited from its class.""" - try: - doc = object.__getattribute__(obj, '__doc__') - if doc is None: - return None - if obj is not type: - typedoc = type(obj).__doc__ - if isinstance(typedoc, str) and typedoc == doc: - return None - return doc - except AttributeError: - return None - def _getdoc(object): - """Get the documentation string for an object. - - All tabs are expanded to spaces. To clean up docstrings that are - indented to line up with blocks of code, any whitespace than can be - uniformly removed from the second line onwards is removed.""" - doc = _getowndoc(object) - if doc is None: - try: - doc = _finddoc(object) - except (AttributeError, TypeError): - return None - if not isinstance(doc, str): - return None - return inspect.cleandoc(doc) + return inspect.getdoc(object, + fallback_to_class_doc=False, + inherit_class_doc=False) def getdoc(object): """Get the doc string or comments for an object.""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d42f2dbff99..24fd4a2fa62 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -688,10 +688,37 @@ def test_getdoc_inherited(self): self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited_class_doc(self): + class A: + """Common base class""" + class B(A): + pass + + a = A() + self.assertEqual(inspect.getdoc(A), 'Common base class') + self.assertEqual(inspect.getdoc(A, inherit_class_doc=False), + 'Common base class') + self.assertEqual(inspect.getdoc(a), 'Common base class') + self.assertIsNone(inspect.getdoc(a, fallback_to_class_doc=False)) + a.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(a, fallback_to_class_doc=False), + 'Instance') + + b = B() + self.assertEqual(inspect.getdoc(B), 'Common base class') + self.assertIsNone(inspect.getdoc(B, inherit_class_doc=False)) + self.assertIsNone(inspect.getdoc(b)) + self.assertIsNone(inspect.getdoc(b, fallback_to_class_doc=False)) + b.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc self.assertEqual(finddoc(int), int.__doc__) + self.assertIsNone(finddoc(int, search_in_class=False)) self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) diff --git a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst new file mode 100644 index 00000000000..d0c8e2d705c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst @@ -0,0 +1,2 @@ +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. From 9e7340cd3b5531784291088b504882cfb4d4c78c Mon Sep 17 00:00:00 2001 From: J Berg Date: Tue, 11 Nov 2025 22:09:58 +0000 Subject: [PATCH 136/417] gh-139462: Make the ProcessPoolExecutor BrokenProcessPool exception report which child process terminated (GH-139486) Report which process terminated as cause of BPE --- Doc/whatsnew/3.15.rst | 10 ++++++++++ Lib/concurrent/futures/process.py | 18 ++++++++++++++++-- .../test_process_pool.py | 15 +++++++++++++++ ...5-10-02-22-29-00.gh-issue-139462.VZXUHe.rst | 3 +++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index ecab0d03e10..c543b6e6c2a 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -369,6 +369,16 @@ collections.abc :mod:`!collections.abc` module. +concurrent.futures +------------------ + +* Improved error reporting when a child process in a + :class:`concurrent.futures.ProcessPoolExecutor` terminates abruptly. + The resulting traceback will now tell you the PID and exit code of the + terminated process. + (Contributed by Jonathan Berg in :gh:`139486`.) + + dataclasses ----------- diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a14650bf5fa..a42afa68efc 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -474,9 +474,23 @@ def _terminate_broken(self, cause): bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") + cause_str = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_str = ''.join(cause) + else: + # No cause known, so report any processes that have + # terminated with nonzero exit codes, e.g. from a + # segfault. Multiple may terminate simultaneously, + # so include all of them in the traceback. + errors = [] + for p in self.processes.values(): + if p.exitcode is not None and p.exitcode != 0: + errors.append(f"Process {p.pid} terminated abruptly " + f"with exit code {p.exitcode}") + if errors: + cause_str = "\n".join(errors) + if cause_str: + bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 9685f980119..731419a48bd 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -106,6 +106,21 @@ def test_traceback(self): self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + def test_traceback_when_child_process_terminates_abruptly(self): + # gh-139462 enhancement - BrokenProcessPool exceptions + # should describe which process terminated. + exit_code = 99 + with self.executor_type(max_workers=1) as executor: + future = executor.submit(os._exit, exit_code) + with self.assertRaises(BrokenProcessPool) as bpe: + future.result() + + cause = bpe.exception.__cause__ + self.assertIsInstance(cause, futures.process._RemoteTraceback) + self.assertIn( + f"terminated abruptly with exit code {exit_code}", cause.tb + ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') def test_ressources_gced_in_workers(self): diff --git a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst new file mode 100644 index 00000000000..390a6124386 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst @@ -0,0 +1,3 @@ +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID +and exit code of the terminated process. Contributed by Jonathan Berg. From 4359706ac8d5589fc37e2f1460a0d07a2319df15 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 12 Nov 2025 00:27:13 +0200 Subject: [PATCH 137/417] gh-120950: Fix overflow in math.log() with large int-like argument (GH-121011) Handling of arbitrary large int-like argument is now consistent with handling arbitrary large int arguments. --- Lib/test/test_math.py | 59 ++++++++++++++ ...-06-26-16-16-43.gh-issue-121011.qW54eh.rst | 2 + Modules/mathmodule.c | 80 ++++++++++++------- 3 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index ddeb8ad7cd6..68f41a2e620 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -189,6 +189,22 @@ def __init__(self, value): def __index__(self): return self.value +class IndexableFloatLike: + def __init__(self, float_value, index_value): + self.float_value = float_value + self.index_value = index_value + + def __float__(self): + if isinstance(self.float_value, BaseException): + raise self.float_value + return self.float_value + + def __index__(self): + if isinstance(self.index_value, BaseException): + raise self.index_value + return self.index_value + + class BadDescr: def __get__(self, obj, objtype=None): raise ValueError @@ -1192,13 +1208,32 @@ def testLog(self): self.ftest('log(10**40, 10**20)', math.log(10**40, 10**20), 2) self.ftest('log(10**1000)', math.log(10**1000), 2302.5850929940457) + self.ftest('log(10**2000, 10**1000)', math.log(10**2000, 10**1000), 2) + self.ftest('log(MyIndexable(32), MyIndexable(2))', + math.log(MyIndexable(32), MyIndexable(2)), 5) + self.ftest('log(MyIndexable(10**1000))', + math.log(MyIndexable(10**1000)), + 2302.5850929940457) + self.ftest('log(MyIndexable(10**2000), MyIndexable(10**1000))', + math.log(MyIndexable(10**2000), MyIndexable(10**1000)), + 2) + self.assertRaises(ValueError, math.log, 0.0) + self.assertRaises(ValueError, math.log, 0) + self.assertRaises(ValueError, math.log, MyIndexable(0)) self.assertRaises(ValueError, math.log, -1.5) + self.assertRaises(ValueError, math.log, -1) + self.assertRaises(ValueError, math.log, MyIndexable(-1)) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) + self.assertEqual(math.log(IndexableFloatLike(math.e, 10**1000)), 1.0) + self.assertAlmostEqual(math.log(IndexableFloatLike(OverflowError(), 10**1000)), + 2302.5850929940457) + def testLog1p(self): self.assertRaises(TypeError, math.log1p) for n in [2, 2**90, 2**300]: @@ -1214,16 +1249,28 @@ def testLog2(self): self.assertEqual(math.log2(1), 0.0) self.assertEqual(math.log2(2), 1.0) self.assertEqual(math.log2(4), 2.0) + self.assertEqual(math.log2(MyIndexable(4)), 2.0) # Large integer values self.assertEqual(math.log2(2**1023), 1023.0) self.assertEqual(math.log2(2**1024), 1024.0) self.assertEqual(math.log2(2**2000), 2000.0) + self.assertEqual(math.log2(MyIndexable(2**2000)), 2000.0) + self.assertRaises(ValueError, math.log2, 0.0) + self.assertRaises(ValueError, math.log2, 0) + self.assertRaises(ValueError, math.log2, MyIndexable(0)) self.assertRaises(ValueError, math.log2, -1.5) + self.assertRaises(ValueError, math.log2, -1) + self.assertRaises(ValueError, math.log2, MyIndexable(-1)) + self.assertRaises(ValueError, math.log2, -2**2000) + self.assertRaises(ValueError, math.log2, MyIndexable(-2**2000)) self.assertRaises(ValueError, math.log2, NINF) self.assertTrue(math.isnan(math.log2(NAN))) + self.assertEqual(math.log2(IndexableFloatLike(8.0, 2**2000)), 3.0) + self.assertEqual(math.log2(IndexableFloatLike(OverflowError(), 2**2000)), 2000.0) + @requires_IEEE_754 # log2() is not accurate enough on Mac OS X Tiger (10.4) @support.requires_mac_ver(10, 5) @@ -1239,12 +1286,24 @@ def testLog10(self): self.ftest('log10(1)', math.log10(1), 0) self.ftest('log10(10)', math.log10(10), 1) self.ftest('log10(10**1000)', math.log10(10**1000), 1000.0) + self.ftest('log10(MyIndexable(10))', math.log10(MyIndexable(10)), 1) + self.ftest('log10(MyIndexable(10**1000))', + math.log10(MyIndexable(10**1000)), 1000.0) + self.assertRaises(ValueError, math.log10, 0.0) + self.assertRaises(ValueError, math.log10, 0) + self.assertRaises(ValueError, math.log10, MyIndexable(0)) self.assertRaises(ValueError, math.log10, -1.5) + self.assertRaises(ValueError, math.log10, -1) + self.assertRaises(ValueError, math.log10, MyIndexable(-1)) self.assertRaises(ValueError, math.log10, -10**1000) + self.assertRaises(ValueError, math.log10, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log10, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + self.assertEqual(math.log10(IndexableFloatLike(100.0, 10**1000)), 2.0) + self.assertEqual(math.log10(IndexableFloatLike(OverflowError(), 10**1000)), 1000.0) + @support.bigmemtest(2**32, memuse=0.2) def test_log_huge_integer(self, size): v = 1 << size diff --git a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst new file mode 100644 index 00000000000..aee7fe2bcb5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst @@ -0,0 +1,2 @@ +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 82846843cfb..de1886451ed 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -57,6 +57,7 @@ raised for division by zero and mod by zero. #endif #include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_import.h" // _PyImport_SetModuleString() @@ -1577,44 +1578,63 @@ math_modf_impl(PyObject *module, double x) However, intermediate overflow is possible for an int if the number of bits in that int is larger than PY_SSIZE_T_MAX. */ +static PyObject* +loghelper_int(PyObject* arg, double (*func)(double)) +{ + /* If it is int, do it ourselves. */ + double x, result; + int64_t e; + + /* Negative or zero inputs give a ValueError. */ + if (!_PyLong_IsPositive((PyLongObject *)arg)) { + PyErr_SetString(PyExc_ValueError, + "expected a positive input"); + return NULL; + } + + x = PyLong_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception and continue. */ + PyErr_Clear(); + x = _PyLong_Frexp((PyLongObject *)arg, &e); + assert(!PyErr_Occurred()); + /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ + result = fma(func(2.0), (double)e, func(x)); + } + else + /* Successfully converted x to a double. */ + result = func(x); + return PyFloat_FromDouble(result); +} + static PyObject* loghelper(PyObject* arg, double (*func)(double)) { /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { - double x, result; - int64_t e; - - /* Negative or zero inputs give a ValueError. */ - if (!_PyLong_IsPositive((PyLongObject *)arg)) { - /* The input can be an arbitrary large integer, so we - don't include it's value in the error message. */ - PyErr_SetString(PyExc_ValueError, - "expected a positive input"); + return loghelper_int(arg, func); + } + /* Else let libm handle it by itself. */ + PyObject *res = math_1(arg, func, 0, "expected a positive input, got %s"); + if (res == NULL && + PyErr_ExceptionMatches(PyExc_OverflowError) && + PyIndex_Check(arg)) + { + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception, convert to + integer and continue. */ + PyErr_Clear(); + arg = _PyNumber_Index(arg); + if (arg == NULL) { return NULL; } - - x = PyLong_AsDouble(arg); - if (x == -1.0 && PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) - return NULL; - /* Here the conversion to double overflowed, but it's possible - to compute the log anyway. Clear the exception and continue. */ - PyErr_Clear(); - x = _PyLong_Frexp((PyLongObject *)arg, &e); - assert(e >= 0); - assert(!PyErr_Occurred()); - /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ - result = fma(func(2.0), (double)e, func(x)); - } - else - /* Successfully converted x to a double. */ - result = func(x); - return PyFloat_FromDouble(result); + res = loghelper_int(arg, func); + Py_DECREF(arg); } - - /* Else let libm handle it by itself. */ - return math_1(arg, func, 0, "expected a positive input, got %s"); + return res; } From f5c2a41f9a6b3be95c5be9dbae0a4a3342d356dc Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 07:47:57 +0800 Subject: [PATCH 138/417] gh-138775: fix handle `python -m base64` stdin correct with EOF signal (GH-138776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: handle stdin correct with EOF single. * fix: flollow the comments when pipe stdin use buffer * Apply suggestions from code review * fix: apply review comments in Lib/base64.py * fix: address comments * Reword comment and NEWS entry. --------- Signed-off-by: yihong0618 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Peter Bierma Co-authored-by: Gregory P. Smith --- Lib/base64.py | 9 ++++++++- .../2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst diff --git a/Lib/base64.py b/Lib/base64.py index cfc57626c40..f95132a4274 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -604,7 +604,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst new file mode 100644 index 00000000000..455c1a9925a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst @@ -0,0 +1,2 @@ +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from a +terminal so that it properly notices EOF. From 0d7b48a8f5de5c1c6d57e1cf7194b6fb222d92e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Wed, 12 Nov 2025 01:03:14 +0100 Subject: [PATCH 139/417] gh-137952: update `csv.Sniffer().has_header()` docs to describe the actual off-by-onish behavior (GH-137953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * checks 21, not 20 * Say "header" instead of "first row" to disambiguate per review. --------- Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Maurycy Pawłowski-Wieroński --- Doc/library/csv.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 3ea7cd210f7..4a033d823e6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -295,8 +295,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: From 0e88be6f55f35ab045e57f9f869b893c15dcc099 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:32:26 +0100 Subject: [PATCH 140/417] gh-138621: Increase test coverage for csv.DictReader and csv.Sniffer (GH-138622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Increase test coverage for csv.DictReader and csv.Sniffer Previously there were no tests for the DictReader fieldnames setter, the case where a StopIteration was encountered when trying to determine the fieldnames from the content or the case where Sniffer could not find a delimiter. * Revert whitespace change to comment * Add a test that csv.Sniffer.has_header checks up to 20 rows * Replace name and age with letter and offset Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * Address review comment --------- Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_csv.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 6be6a7ae222..df79840088a 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -918,6 +918,14 @@ def test_dict_reader_fieldnames_accepts_list(self): reader = csv.DictReader(f, fieldnames) self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_reader_set_fieldnames(self): + fieldnames = ["a", "b", "c"] + f = StringIO() + reader = csv.DictReader(f) + self.assertIsNone(reader.fieldnames) + reader.fieldnames = fieldnames + self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_writer_fieldnames_rejects_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() @@ -933,6 +941,7 @@ def test_dict_writer_fieldnames_accepts_list(self): def test_dict_reader_fieldnames_is_optional(self): f = StringIO() reader = csv.DictReader(f, fieldnames=None) + self.assertIsNone(reader.fieldnames) def test_read_dict_fields(self): with TemporaryFile("w+", encoding="utf-8") as fileobj: @@ -1353,6 +1362,19 @@ class TestSniffer(unittest.TestCase): ghi\0jkl """ + sample15 = "\n\n\n" + sample16 = "abc\ndef\nghi" + + sample17 = ["letter,offset"] + sample17.extend(f"{chr(ord('a') + i)},{i}" for i in range(20)) + sample17.append("v,twenty_one") # 'u' was skipped + sample17 = '\n'.join(sample17) + + sample18 = ["letter,offset"] + sample18.extend(f"{chr(ord('a') + i)},{i}" for i in range(21)) + sample18.append("v,twenty_one") # 'u' was not skipped + sample18 = '\n'.join(sample18) + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1374,6 +1396,11 @@ def test_has_header_regex_special_delimiter(self): self.assertIs(sniffer.has_header(self.sample8), False) self.assertIs(sniffer.has_header(self.header2 + self.sample8), True) + def test_has_header_checks_20_rows(self): + sniffer = csv.Sniffer() + self.assertFalse(sniffer.has_header(self.sample17)) + self.assertTrue(sniffer.has_header(self.sample18)) + def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): @@ -1423,6 +1450,10 @@ def test_delimiters(self): self.assertEqual(dialect.quotechar, "'") dialect = sniffer.sniff(self.sample14) self.assertEqual(dialect.delimiter, '\0') + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample15) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample16) def test_doublequote(self): sniffer = csv.Sniffer() From df6676549cd67c7b83111c6fce7c546270604aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Cabello=20Jim=C3=A9nez?= <33024690+acabelloj@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:36:43 +0100 Subject: [PATCH 141/417] gh-137928: remove redundant size validation in multiprocessing.heap (GH-137929) remove redundant size check, malloc does it --------- Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/heap.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py index 6217dfe1268..5c835648395 100644 --- a/Lib/multiprocessing/heap.py +++ b/Lib/multiprocessing/heap.py @@ -324,10 +324,6 @@ class BufferWrapper(object): _heap = Heap() def __init__(self, size): - if size < 0: - raise ValueError("Size {0:n} out of range".format(size)) - if sys.maxsize <= size: - raise OverflowError("Size {0:n} too large".format(size)) block = BufferWrapper._heap.malloc(size) self._state = (block, size) util.Finalize(self, BufferWrapper._heap.free, args=(block,)) From 9ce99c6c1901705238e4cb3ce81eb6f499e7b4f4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 12 Nov 2025 00:53:21 +0000 Subject: [PATCH 142/417] GH-137618: Require Python 3.10 to Python 3.15 for PYTHON_FOR_REGEN (GH-137619) * Require Python 3.11 to Python 3.15 for PYTHON_FOR_REGEN * NEWS * keep allowing python 3.10 --------- Co-authored-by: Gregory P. Smith --- .../next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst | 2 ++ configure | 2 +- configure.ac | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst diff --git a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst new file mode 100644 index 00000000000..0b56c4c8f68 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst @@ -0,0 +1,2 @@ +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. +Patch by Adam Turner. diff --git a/configure b/configure index 8463b5b5e4a..eeb24c1d844 100755 --- a/configure +++ b/configure @@ -3818,7 +3818,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 diff --git a/configure.ac b/configure.ac index df94ae25e63..92adc44da0d 100644 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST([PYTHON_FOR_REGEN]) From fbebca289d811669fc1980e3a135325b8542a846 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 12 Nov 2025 09:59:48 +0500 Subject: [PATCH 143/417] GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268) --- Modules/_threadmodule.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cc8277c5783..0e22c7bd386 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,6 +41,7 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; + PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -48,6 +49,7 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; + PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -767,7 +769,6 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -999,6 +1000,10 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } +static PyMemberDef lock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, + {NULL} +}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1034,8 +1039,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, + {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, - {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1043,8 +1048,7 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -1059,7 +1063,6 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1319,6 +1322,11 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ +static PyMemberDef rlock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, + {NULL} +}; + static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1339,10 +1347,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, + {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1350,7 +1358,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + Py_TPFLAGS_IMMUTABLETYPE), .slots = rlock_type_slots, }; From ef474cfafbdf3aa383fb1334a7ab95cef9834ced Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 10:47:38 +0530 Subject: [PATCH 144/417] gh-103847: fix cancellation safety of `asyncio.create_subprocess_exec` (#140805) --- Lib/asyncio/base_subprocess.py | 11 +++++ Lib/test/test_asyncio/test_subprocess.py | 40 ++++++++++++++++++- ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d40af422e61..321a4e5d5d1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -213,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -256,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3a17c169c34..bf301740741 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -11,7 +11,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -879,6 +879,44 @@ async def main(): self.loop.run_until_complete(main()) + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst new file mode 100644 index 00000000000..e14af7d9708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst @@ -0,0 +1 @@ +Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. From f1b7961ccfa050e9c80622fff1b3cdada46f9aab Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 12:51:43 +0530 Subject: [PATCH 145/417] GH-116946: revert eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141448) Revert "GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268)" This reverts commit fbebca289d811669fc1980e3a135325b8542a846. --- Modules/_threadmodule.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 0e22c7bd386..cc8277c5783 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,7 +41,6 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; - PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -49,7 +48,6 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; - PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -769,6 +767,7 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1000,10 +999,6 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } -static PyMemberDef lock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, - {NULL} -}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1039,8 +1034,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, - {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1048,7 +1043,8 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -1063,6 +1059,7 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1322,11 +1319,6 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ -static PyMemberDef rlock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, - {NULL} -}; - static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1347,10 +1339,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, - {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, + {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1358,7 +1350,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; From 35908265b09ac39b67116bfdfe8a053be09e6d8f Mon Sep 17 00:00:00 2001 From: Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com> Date: Wed, 12 Nov 2025 09:20:55 +0100 Subject: [PATCH 146/417] gh-75593: Add support of bytes and path-like paths in wave.open() (GH-140951) --- Doc/library/wave.rst | 9 ++++++-- Lib/test/test_wave.py | 22 +++++++++++++++++++ Lib/wave.py | 5 +++-- ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index a3f5bfd5e2f..7ff2c97992c 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -25,8 +25,9 @@ The :mod:`wave` module defines the following function and exception: .. function:: open(file, mode=None) - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + If *file* is a string, a :term:`path-like object` or a + :term:`bytes-like object` open the file by that name, otherwise treat it as + a file-like object. *mode* can be: ``'rb'`` Read only mode. @@ -52,6 +53,10 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. + .. versionchanged:: 3.15 + Added support for :term:`path-like objects ` + and :term:`bytes-like objects `. + .. exception:: Error An error raised when something is impossible because it violates the WAV diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 226b1aa84bd..4c21f165537 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,9 +1,11 @@ import unittest from test import audiotests from test import support +from test.support.os_helper import FakePath import io import os import struct +import tempfile import sys import wave @@ -206,5 +208,25 @@ def test_open_in_write_raises(self): self.assertIsNone(cm.unraisable) +class WaveOpen(unittest.TestCase): + def test_open_pathlike(self): + """It is possible to use `wave.read` and `wave.write` with a path-like object""" + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + cases = ( + FakePath(fp.name), + FakePath(os.fsencode(fp.name)), + os.fsencode(fp.name), + ) + for fake_path in cases: + with self.subTest(fake_path): + with wave.open(fake_path, 'wb') as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(44100) + + with wave.open(fake_path, 'rb') as f: + pass + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index 5af745e2217..056bd6aab7f 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -69,6 +69,7 @@ from collections import namedtuple import builtins +import os import struct import sys @@ -274,7 +275,7 @@ def initfp(self, file): def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'rb') self._i_opened_the_file = f # else, assume it is an open file object already @@ -431,7 +432,7 @@ class Wave_write: def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'wb') self._i_opened_the_file = f try: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst new file mode 100644 index 00000000000..9a31af9c110 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst @@ -0,0 +1 @@ +Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. From 909f76dab91f028edd2ae7bd589d3975996de9e1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 12 Nov 2025 09:42:56 +0100 Subject: [PATCH 147/417] gh-141376: Rename _AsyncioDebug to _Py_AsyncioDebug (GH-141391) --- Modules/_asynciomodule.c | 4 ++-- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 1f58b1fb350..9b2b7011244 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -119,7 +119,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; -GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _AsyncioDebug) +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), @@ -4338,7 +4338,7 @@ module_init(asyncio_state *state) goto fail; } - state->debug_offsets = &_AsyncioDebug; + state->debug_offsets = &_Py_AsyncioDebug; Py_DECREF(module); return 0; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8b73189fb07..11a3cd794ff 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -56,7 +56,7 @@ Python/pyhash.c - _Py_HashSecret - Python/parking_lot.c - buckets - ## data needed for introspecting asyncio state from debuggers and profilers -Modules/_asynciomodule.c - _AsyncioDebug - +Modules/_asynciomodule.c - _Py_AsyncioDebug - ################################## From 6f988b08d122e44848e89c04ad1e10c25d072cc7 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:37:48 -0800 Subject: [PATCH 148/417] gh-85524: Raise "UnsupportedOperation" on FileIO.readall (#141214) io.UnsupportedOperation is a subclass of OSError and recommended by io.IOBase for this case; matches other read methods on io.FileIO. --- Lib/test/test_io/test_general.py | 1 + .../2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 +++ Modules/_io/clinic/fileio.c.h | 14 +++++++++----- Modules/_io/fileio.c | 13 ++++++++++--- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index a1cdd6876c2..f0677b01ea5 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -125,6 +125,7 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readall) self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: self.assertRaises(exc, fp.write, b"blah") diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst new file mode 100644 index 00000000000..3e4fd1a5897 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst @@ -0,0 +1,3 @@ +Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, +to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` +when a file is in "w" mode rather than :exc:`OSError` diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 04870b1c890..96c31ce8d6f 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -277,15 +277,19 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "data is available (EAGAIN is returned before bytes are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ - {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, + {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, static PyObject * -_io_FileIO_readall_impl(fileio *self); +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readall(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_readall_impl((fileio *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "readall() takes no arguments"); + return NULL; + } + return _io_FileIO_readall_impl((fileio *)self, cls); } PyDoc_STRVAR(_io_FileIO_read__doc__, @@ -543,4 +547,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1902fac9e39358aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 2544ff4ea91..5d7741fdd83 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -728,6 +728,9 @@ new_buffersize(fileio *self, size_t currentsize) @permit_long_docstring_body _io.FileIO.readall + cls: defining_class + / + Read all data from the file, returned as bytes. Reads until either there is an error or read() returns size 0 (indicates EOF). @@ -738,8 +741,8 @@ data is available (EAGAIN is returned before bytes are read) returns None. [clinic start generated code]*/ static PyObject * -_io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=10d8b2ec403302dc]*/ +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ { Py_off_t pos, end; PyBytesWriter *writer; @@ -750,6 +753,10 @@ _io_FileIO_readall_impl(fileio *self) if (self->fd < 0) { return err_closed(); } + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { end = (Py_off_t)self->stat_atopen->st_size; @@ -873,7 +880,7 @@ _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) } if (size < 0) - return _io_FileIO_readall_impl(self); + return _io_FileIO_readall_impl(self, cls); if (size > _PY_READ_MAX) { size = _PY_READ_MAX; From 20f53df07d42c495a08c73a3d54b8dd9098a62f0 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 12:50:44 +0300 Subject: [PATCH 149/417] gh-141370: document undefined behavior of Py_ABS() (GH-141439) --- Doc/c-api/intro.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 6e1a9dcb355..c76cc2f70ec 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -121,6 +121,10 @@ complete listing. Return the absolute value of ``x``. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + .. versionadded:: 3.3 .. c:macro:: Py_ALWAYS_INLINE From 7d54374f9c7d91e0ef90c4ad84baf10073cf1d8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:57:05 -0800 Subject: [PATCH 150/417] gh-141311: Avoid assertion in BytesIO.readinto() (GH-141333) Fix error in assertion which causes failure if pos is equal to PY_SSIZE_T_MAX. Fix undefined behavior in read() and readinto() if pos is larger that the size of the underlying buffer. --- Lib/test/test_io/test_memoryio.py | 14 ++++++++++++++ ...025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 ++ Modules/_io/bytesio.c | 16 +++++++++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 63998a86c45..bb023735e21 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -54,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -552,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst new file mode 100644 index 00000000000..bb425ce5df3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst @@ -0,0 +1,2 @@ +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index d6bfb93177c..96611823ab6 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -436,6 +436,13 @@ read_bytes_lock_held(bytesio *self, Py_ssize_t size) return Py_NewRef(self->buf); } + /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX) + is beyond the size of self->buf. Assert above validates size is always in + bounds. When self->pos is out of bounds calling code sets size to 0. */ + if (size == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); @@ -609,11 +616,14 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) n = self->string_size - self->pos; if (len > n) { len = n; - if (len < 0) - len = 0; + if (len < 0) { + /* gh-141311: Avoid undefined behavior when self->pos (limit + PY_SSIZE_T_MAX) points beyond the size of self->buf. */ + return PyLong_FromSsize_t(0); + } } - assert(self->pos + len < PY_SSIZE_T_MAX); + assert(self->pos + len <= PY_SSIZE_T_MAX); assert(len >= 0); memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; From 23d85a2a3fb029172ea15c6e596f64f8c2868ed3 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 13:06:29 +0300 Subject: [PATCH 151/417] gh-141042: fix sNaN's packing for mixed floating-point formats (#141107) --- Lib/test/test_capi/test_float.py | 54 +++++++++++++++---- ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 ++ Objects/floatobject.c | 16 ++++-- 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index 983b991b4f1..df7017e6436 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -29,6 +29,23 @@ NAN = float("nan") +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() @@ -202,16 +219,7 @@ def test_pack_unpack_roundtrip_for_nans(self): # HP PA RISC uses 0 for quiet, see: # https://en.wikipedia.org/wiki/NaN#Encoding signaling = 1 - quiet = int(not signaling) - if size == 8: - payload = random.randint(signaling, 0x7ffffffffffff) - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + payload - elif size == 4: - payload = random.randint(signaling, 0x3fffff) - i = (sign << 31) + (0xff << 23) + (quiet << 22) + payload - elif size == 2: - payload = random.randint(signaling, 0x1ff) - i = (sign << 15) + (0x1f << 10) + (quiet << 9) + payload + i = make_nan(size, sign, not signaling) data = bytes.fromhex(f'{i:x}') for endian in (BIG_ENDIAN, LITTLE_ENDIAN): with self.subTest(data=data, size=size, endian=endian): @@ -221,6 +229,32 @@ def test_pack_unpack_roundtrip_for_nans(self): self.assertTrue(math.isnan(value)) self.assertEqual(data1, data2) + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst new file mode 100644 index 00000000000..22a1aa1f405 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst @@ -0,0 +1,3 @@ +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1fefb12803e..ef613efe4e7 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2030,6 +2030,10 @@ PyFloat_Pack2(double x, char *data, int le) memcpy(&v, &x, sizeof(v)); v &= 0xffc0000000000ULL; bits = (unsigned short)(v >> 42); /* NaN's type & payload */ + /* set qNaN if no payload */ + if (!bits) { + bits |= (1<<9); + } } else { sign = (x < 0.0); @@ -2202,16 +2206,16 @@ PyFloat_Pack4(double x, char *data, int le) if ((v & (1ULL << 51)) == 0) { uint32_t u32; memcpy(&u32, &y, 4); - u32 &= ~(1 << 22); /* make sNaN */ + /* if have payload, make sNaN */ + if (u32 & 0x3fffff) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); } #else uint32_t u32; memcpy(&u32, &y, 4); - if ((v & (1ULL << 51)) == 0) { - u32 &= ~(1 << 22); - } /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2222,6 +2226,10 @@ PyFloat_Pack4(double x, char *data, int le) /* add payload */ u32 -= (u32 & 0x3fffff); u32 += (uint32_t)((v & 0x7ffffffffffffULL) >> 29); + /* if have payload, make sNaN */ + if ((v & (1ULL << 51)) == 0 && (u32 & 0x3fffff)) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); #endif From 70748bdbea872a84dd8eadad9b48c73e218d2e1f Mon Sep 17 00:00:00 2001 From: Jacob Austin Lincoln <99031153+lincolnj1@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:07:21 -0800 Subject: [PATCH 152/417] gh-131116: Fix inspect.getdoc() to work with cached_property objects (GH-131165) --- Doc/library/inspect.rst | 3 ++ Lib/inspect.py | 6 +++ Lib/test/test_inspect/inspect_fodder3.py | 39 +++++++++++++++++++ Lib/test/test_inspect/test_inspect.py | 20 ++++++++++ ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 + 5 files changed, 70 insertions(+) create mode 100644 Lib/test/test_inspect/inspect_fodder3.py create mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index aff53b78c4a..13a352cbdb2 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -639,6 +639,9 @@ Retrieving source code .. versionchanged:: next Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overriden. + .. function:: getcomments(object) diff --git a/Lib/inspect.py b/Lib/inspect.py index bb17848b444..8e7511b3af0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -747,6 +747,12 @@ def _finddoc(obj, *, search_in_class=True): cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None + # Should be tested before ismethoddescriptor() + elif isinstance(obj, functools.cached_property): + name = obj.attrname + cls = _findclass(obj.func) + if cls is None or getattr(cls, name) is not obj: + return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 00000000000..ea2481edf93 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 24fd4a2fa62..dd3b7d9c5b4 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -46,6 +46,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -714,6 +715,25 @@ class B(A): b.__doc__ = 'Instance' self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst new file mode 100644 index 00000000000..f5e60ab6e8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst @@ -0,0 +1,2 @@ +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. From c6f3dd6a506a9bb1808c070e5ef5cf345a3bedc8 Mon Sep 17 00:00:00 2001 From: Rani Pinchuk <33353578+rani-pinchuk@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:35:01 +0100 Subject: [PATCH 153/417] gh-98896: resource_tracker: use json&base64 to allow arbitrary shared memory names (GH-138473) --- Lib/multiprocessing/resource_tracker.py | 60 ++++++++++++++++--- Lib/test/_test_multiprocessing.py | 43 +++++++++++++ ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 + 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 38fcaed48fa..b0f9099f4a5 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,6 +15,7 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys @@ -22,6 +23,8 @@ import warnings from collections import deque +import json + from . import spawn from . import util @@ -196,6 +199,17 @@ def _launch(self): finally: os.close(r) + def _make_probe_message(self): + """Return a JSON-encoded probe message.""" + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: @@ -207,7 +221,7 @@ def _ensure_running_and_write(self, msg=None): if self._fd is not None: # resource tracker was launched before, is it still running? if msg is None: - to_send = b'PROBE:0:noop\n' + to_send = self._make_probe_message() else: to_send = msg try: @@ -234,7 +248,7 @@ def _check_alive(self): try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -253,11 +267,25 @@ def _write(self, msg): assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" def _send(self, cmd, name, rtype): - msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" self._ensure_running_and_write(msg) @@ -290,7 +318,23 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 850744e47d0..0f9c5c22225 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -7364,3 +7364,46 @@ def test_forkpty(self): res = assert_python_failure("-c", code, PYTHONWARNINGS='error') self.assertIn(b'DeprecationWarning', res.err) self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + import time + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """) + + rc, out, err = assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst new file mode 100644 index 00000000000..6831499c0af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst @@ -0,0 +1,2 @@ +Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. +Patch by Rani Pinchuk. From e2026731f5680022bd016b8b5ca5841c82e9574c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 15:44:49 +0300 Subject: [PATCH 154/417] gh-141004: soft-deprecate Py_INFINITY macro (#141033) Co-authored-by: Victor Stinner --- Doc/c-api/conversion.rst | 2 +- Doc/c-api/float.rst | 7 +++++-- Doc/whatsnew/3.14.rst | 2 +- Doc/whatsnew/3.15.rst | 4 ++++ Include/floatobject.h | 16 +++++++-------- Include/internal/pycore_pymath.h | 6 +++--- Include/pymath.h | 3 ++- ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 + Modules/cmathmodule.c | 6 +++--- Modules/mathmodule.c | 20 +++++++++---------- Objects/complexobject.c | 8 ++++---- Objects/floatobject.c | 2 +- Python/pystrtod.c | 4 ++-- 13 files changed, 45 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 533e5460da8..a18bbf4e0e3 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index eae4792af7d..b6020533a2b 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -83,8 +83,11 @@ Floating-Point Objects This macro expands a to constant expression of type :c:expr:`double`, that represents the positive infinity. - On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from - the C11 standard ```` header. + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. deprecated:: 3.15 + The macro is soft deprecated. .. c:macro:: Py_NAN diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1a2fbda0c4c..9459b73bcb5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3045,7 +3045,7 @@ Deprecated C APIs ----------------- * The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. - Use :c:macro:`!Py_INFINITY` instead. + Use :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`120026`.) * The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c543b6e6c2a..f0fd49c9033 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1095,6 +1095,10 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5..814337b070a 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68..4fcac3aab8b 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/pymath.h b/Include/pymath.h index 0f9f0f3b299..7cfe441365d 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -45,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst new file mode 100644 index 00000000000..a054f8eda6f --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index a4ea5557a6a..aee3e4f343d 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_INFINITY +#define INF INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -1186,11 +1186,11 @@ cmath_exec(PyObject *mod) if (PyModule_Add(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(mod, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } - Py_complex infj = {0.0, Py_INFINITY}; + Py_complex infj = {0.0, INFINITY}; if (PyModule_Add(mod, "infj", PyComplex_FromCComplex(infj)) < 0) { return -1; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index de1886451ed..11c46c987e1 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -395,7 +395,7 @@ m_tgamma(double x) if (x == 0.0) { errno = EDOM; /* tgamma(+-0.0) = +-inf, divide-by-zero */ - return copysign(Py_INFINITY, x); + return copysign(INFINITY, x); } /* integer arguments */ @@ -426,7 +426,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_INFINITY; + return INFINITY; } } @@ -490,14 +490,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_INFINITY; /* lgamma(+-inf) = +inf */ + return INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_INFINITY; /* integers n <= 0 */ + return INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -633,7 +633,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log(0) = -inf */ + return -INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -676,7 +676,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ + return -INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -692,7 +692,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log10(0) = -inf */ + return -INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -1500,7 +1500,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_INFINITY, x); + r = copysign(INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2983,7 +2983,7 @@ math_ulp_impl(PyObject *module, double x) if (isinf(x)) { return x; } - double inf = Py_INFINITY; + double inf = INFINITY; double x2 = nextafter(x, inf); if (isinf(x2)) { /* special case: x is the largest positive representable float */ @@ -3007,7 +3007,7 @@ math_exec(PyObject *module) if (PyModule_Add(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(module, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } if (PyModule_Add(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 6247376a0e6..3612c2699a5 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -139,8 +139,8 @@ _Py_c_prod(Py_complex z, Py_complex w) recalc = 1; } if (recalc) { - r.real = Py_INFINITY*(a*c - b*d); - r.imag = Py_INFINITY*(a*d + b*c); + r.real = INFINITY*(a*c - b*d); + r.imag = INFINITY*(a*d + b*c); } } @@ -229,8 +229,8 @@ _Py_c_quot(Py_complex a, Py_complex b) { const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); - r.real = Py_INFINITY * (x*b.real + y*b.imag); - r.imag = Py_INFINITY * (y*b.real - x*b.imag); + r.real = INFINITY * (x*b.real + y*b.imag); + r.imag = INFINITY * (y*b.real - x*b.imag); } else if ((isinf(abs_breal) || isinf(abs_bimag)) && isfinite(a.real) && isfinite(a.imag)) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index ef613efe4e7..78006783c6e 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2415,7 +2415,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_INFINITY : Py_INFINITY; + return sign ? -INFINITY : INFINITY; } else { /* NaN */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 7b74f613ed5..e8aca939d1f 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_INFINITY : Py_INFINITY; + retval = negate ? -INFINITY : INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python + if overflow_exception is NULL then +-INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. From f963864cb54c2e7364b2c850485c6bf25479f6f2 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 20:45:43 +0800 Subject: [PATCH 155/417] gh-141464: a typo in profiling sampling when can not run warning in linux (#141465) --- Lib/profiling/sampling/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/profiling/sampling/__main__.py b/Lib/profiling/sampling/__main__.py index a76ca62e2cd..cd1425b8b9c 100644 --- a/Lib/profiling/sampling/__main__.py +++ b/Lib/profiling/sampling/__main__.py @@ -15,7 +15,7 @@ """ LINUX_PERMISSION_ERROR = """ -🔒 Tachyon was unable to acess process memory. This could be because tachyon +🔒 Tachyon was unable to access process memory. This could be because tachyon has insufficient privileges (the required capability is CAP_SYS_PTRACE). Unprivileged processes cannot trace processes that they cannot send signals to or those running set-user-ID/set-group-ID programs, for security reasons. From 88aeff8eabefdc13b6fb29edb3cde618f743a034 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:22:01 +0000 Subject: [PATCH 156/417] gh-87710: Update mime type for ``.ai`` (#141239) --- Doc/whatsnew/3.15.rst | 4 +++- Lib/mimetypes.py | 2 +- Lib/test/test_mimetypes.py | 1 + .../Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index f0fd49c9033..c6089f63dee 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -472,7 +472,9 @@ mimetypes * Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. - (Contributed by Charlie Lin in :gh:`140165`) + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) mmap diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index d6896fc4042..42477713c78 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -497,9 +497,9 @@ def _default_mime_types(): '.oda' : 'application/oda', '.ogx' : 'application/ogg', '.pdf' : 'application/pdf', + '.ai' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', - '.ai' : 'application/postscript', '.eps' : 'application/postscript', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 746984ec0ca..73414498359 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -229,6 +229,7 @@ def check_extensions(): ("application/octet-stream", ".bin"), ("application/gzip", ".gz"), ("application/ogg", ".ogx"), + ("application/pdf", ".pdf"), ("application/postscript", ".ps"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst new file mode 100644 index 00000000000..62073280e32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst @@ -0,0 +1 @@ +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. From 2ac738d325a6934e39fecb097f43d4d4ed97a2b9 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Wed, 12 Nov 2025 16:20:08 +0100 Subject: [PATCH 157/417] gh-132657: add regression test for `PySet_Contains` with unhashable type (#141411) --- Modules/_testlimitedcapi/set.c | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 35da5fa5f00..34ed6b1d60b 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -155,6 +155,51 @@ test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) return NULL; } +static PyObject * +test_set_contains_does_not_convert_unhashable_key(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // See https://docs.python.org/3/c-api/set.html#c.PySet_Contains + PyObject *outer_set = PySet_New(NULL); + + PyObject *needle = PySet_New(NULL); + if (needle == NULL) { + Py_DECREF(outer_set); + return NULL; + } + + PyObject *num = PyLong_FromLong(42); + if (num == NULL) { + Py_DECREF(outer_set); + Py_DECREF(needle); + return NULL; + } + + if (PySet_Add(needle, num) < 0) { + Py_DECREF(outer_set); + Py_DECREF(needle); + Py_DECREF(num); + return NULL; + } + + int result = PySet_Contains(outer_set, needle); + + Py_DECREF(num); + Py_DECREF(needle); + Py_DECREF(outer_set); + + if (result < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; + } + + PyErr_SetString(PyExc_AssertionError, + "PySet_Contains should have raised TypeError for unhashable key"); + return NULL; +} + static PyMethodDef test_methods[] = { {"set_check", set_check, METH_O}, {"set_checkexact", set_checkexact, METH_O}, @@ -174,6 +219,8 @@ static PyMethodDef test_methods[] = { {"set_clear", set_clear, METH_O}, {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + {"test_set_contains_does_not_convert_unhashable_key", + test_set_contains_does_not_convert_unhashable_key, METH_NOARGS}, {NULL}, }; From f1330b35b8eb43904dfed0656acde80c08d63176 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:37:54 +0000 Subject: [PATCH 158/417] gh-141004: Document `Py_MATH_{E, PI, TAU}` constants (#141373) --- Doc/c-api/float.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b6020533a2b..79de5daaa90 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -99,6 +99,11 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + .. c:macro:: Py_MATH_El High precision (long double) definition of :data:`~math.e` constant. @@ -106,6 +111,11 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + .. c:macro:: Py_MATH_PIl High precision (long double) definition of :data:`~math.pi` constant. @@ -113,6 +123,13 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + .. c:macro:: Py_RETURN_NAN Return :data:`math.nan` from a function. From 9cd5427d9619b96db20d0347a136b3d331af71ae Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 11:38:17 -0500 Subject: [PATCH 159/417] gh-141004: Document `PyType_SUPPORTS_WEAKREFS` (GH-141408) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/type.rst | 26 ++++++++++++++++++++++++++ Doc/c-api/weakref.rst | 8 ++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 29ffeb7c483..b608f815160 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -195,12 +195,14 @@ Type Objects before initialization) and should be paired with :c:func:`PyObject_Free` in :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Creates a new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) Finalize a type object. This should be called on all type objects to finish @@ -217,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -224,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -239,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -246,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -262,6 +268,7 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -281,6 +288,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -295,6 +303,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) Find the first superclass whose module was created from @@ -314,6 +323,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose @@ -332,6 +342,7 @@ Type Objects .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -342,6 +353,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -390,6 +411,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -416,6 +438,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -437,6 +460,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -457,6 +481,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -628,6 +653,7 @@ The following functions and structs are used to create * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 39e4febd3ef..db6ae0a9d4e 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -45,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -57,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) From d162c427904e232fec52d8da759caa1bfa4c01b5 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:09:25 -0800 Subject: [PATCH 160/417] GH-140479: Update JIT builds to use LLVM 21 (#140973) --- .github/workflows/jit.yml | 8 ++++---- ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 + PCbuild/get_externals.bat | 4 ++-- Tools/jit/README.md | 20 +++++++++---------- Tools/jit/_llvm.py | 4 ++-- 5 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69d900091a3..62325250bd3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 20 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst new file mode 100644 index 00000000000..0a615ed1311 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 21 at build time. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 115203cecc8..9d02e2121cc 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,7 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - if "%%b"=="llvm-20.1.8.0" ( + if "%%b"=="llvm-21.1.4.0" ( %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b diff --git a/Tools/jit/README.md b/Tools/jit/README.md index d83b09aab59..dd7deb7b256 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 20 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 20 +sudo ./llvm.sh 21 ``` -Install LLVM 20 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 20 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@20 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,18 +43,18 @@ ### Windows LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=20.1.8 +choco install llvm --version=21.1.0 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f1b0ad3f5db..0b9cb5192f1 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "20" -_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") From fbcac799518e0cb29fcf5f84ed1fa001010b9073 Mon Sep 17 00:00:00 2001 From: Bob Kline Date: Wed, 12 Nov 2025 13:25:23 -0500 Subject: [PATCH 161/417] gh-141412: Use reliable target URL for urllib example (GH-141428) The endpoint used for demonstrating reading URLs is no longer stable. This change substitutes a target over which we have more control. --- Doc/tutorial/stdlib.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 49a3e370a4c..342c1a00193 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') From 1f381a579cc50aa82838de84c2294b4979586bd9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:26:50 -0800 Subject: [PATCH 162/417] Add details about JIT build infrastructure and updating dependencies to `Tools/jit` (#141167) --- Tools/jit/README.md | 3 +++ Tools/jit/jit_infra.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 Tools/jit/jit_infra.md diff --git a/Tools/jit/README.md b/Tools/jit/README.md index dd7deb7b256..c70c0c47d94 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -66,6 +66,9 @@ ## Building The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit). +## Miscellaneous +If you're looking for information on how to update the JIT build dependencies, see [JIT Build Infrastructure](jit_infra.md). + [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md new file mode 100644 index 00000000000..1a954755611 --- /dev/null +++ b/Tools/jit/jit_infra.md @@ -0,0 +1,28 @@ +# JIT Build Infrastructure + +This document includes details about the intricacies of the JIT build infrastructure. + +## Updating LLVM + +When we update LLVM, we need to also update the LLVM release artifact for Windows builds. This is because Windows builds automatically pull prebuilt LLVM binaries in our pipelines (e.g. notice that `.github/workflows/jit.yml` does not explicitly download LLVM or build it from source). + +To update the LLVM release artifact for Windows builds, follow these steps: +1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). +1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). +1. Extract and repackage the tarball with the correct directory structure. For example: + ```bash + tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + ``` + The tarball must contain a top-level directory named `llvm-{version}.0/`. +1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). +1. Create a new release with the updated LLVM artifact. + - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). + - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). + - Upload the asset (you can leave all other fields the same). + +### Other notes +- You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. +- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file From 35ed3e4cedc8aef3936da81a6b64e90374532b13 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 12 Nov 2025 22:04:02 +0300 Subject: [PATCH 163/417] gh-140936: Fix JIT assertion crash at finalization if some generator is alive (GH-140969) --- Lib/test/test_capi/test_opt.py | 19 +++++++++++++++++++ Python/optimizer.c | 8 +++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4e94f62d35e..e65556fb28f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2660,6 +2660,25 @@ def f(): f() + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) def global_identity(x): diff --git a/Python/optimizer.c b/Python/optimizer.c index f44f8a9614b..3b7e2dafab8 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,7 +118,13 @@ _PyOptimizer_Optimize( { _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->jit); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } assert(!interp->compiling); #ifndef Py_GIL_DISABLED interp->compiling = true; From 558936bec1f1e0f8346063a8cb2b2782d085178e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 13 Nov 2025 05:41:26 +0800 Subject: [PATCH 164/417] gh-141442: Add escaping to iOS testbed arguments (#141443) Xcode concatenates the test argument array, losing quoting in the process. --- Apple/testbed/__main__.py | 3 ++- .../Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 42eb60a4c8d..49974cb1428 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -2,6 +2,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -252,7 +253,7 @@ def update_test_plan(testbed_path, platform, args): test_plan = json.load(f) test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ - {"argument": arg} for arg in args + {"argument": shlex.quote(arg)} for arg in args ] with test_plan_path.open("w", encoding="utf-8") as f: diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst new file mode 100644 index 00000000000..073c070413f --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst @@ -0,0 +1 @@ +The iOS testbed now correctly handles test arguments that contain spaces. From dc0987080ed66c662e8e0b24cdb8c179817bd697 Mon Sep 17 00:00:00 2001 From: Michael Cho Date: Wed, 12 Nov 2025 17:16:58 -0500 Subject: [PATCH 165/417] gh-124111: Fix TCL 9 thread detection (GH-128103) --- .../Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 ++ Modules/_tkinter.c | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst new file mode 100644 index 00000000000..8436cd2415d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst @@ -0,0 +1,2 @@ +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c0ed8977d8f..8cea7b59fe7 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -575,8 +575,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; From 26b7df2430cd5a9ee772bfa6ee03a73bd0b11619 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 17:52:56 -0500 Subject: [PATCH 166/417] gh-141004: Document `PyRun_InteractiveOneObject` (GH-141405) --- Doc/c-api/veryhigh.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 916c616dfee..3b07b5fbed5 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -100,6 +100,20 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) + + Read and execute a single statement from a file associated with an + interactive device according to the *flags* argument. The user will be + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. + + Returns ``0`` when the input was + executed successfully, ``-1`` if there was an exception, or an error code + from the :file:`errcode.h` include file distributed as part of Python if + there was a parse error. (Note that :file:`errcode.h` is not included by + :file:`Python.h`, so must be included specifically if needed.) + + .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, @@ -108,17 +122,10 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - Read and execute a single statement from a file associated with an - interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the :term:`filesystem encoding and error handler`. - Returns ``0`` when the input was - executed successfully, ``-1`` if there was an exception, or an error code - from the :file:`errcode.h` include file distributed as part of Python if - there was a parse error. (Note that :file:`errcode.h` is not included by - :file:`Python.h`, so must be included specifically if needed.) - .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) From 781cc68c3c814e46e6a74c3a6a32e0f9f8f7eb11 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Wed, 12 Nov 2025 18:15:16 -0800 Subject: [PATCH 167/417] gh-137109: refactor warning about threads when forking (#141438) * gh-137109: refactor warning about threads when forking This splits the OS API specific functionality to get the number of threads out from the fallback Python method and warning raising code itself. This way the OS APIs can be queried before we've run `os.register_at_fork(after_in_parent=...)` registered functions which themselves may (re)start threads that would otherwise be detected. This is best effort. If the OS APIs are either unavailable or fail, the warning generating code still falls back to looking at the Python threading state after the CPython interpreter world has been restarted and the after_in_parent calls have been made. The common case for most Linux and macOS environments should work today. This also lines up with the existing TODO refactoring, we may choose to expose this API to get the number of OS threads in the `os` module in the future. * NEWS entry * avoid "function-prototype" compiler warning? --- ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 + Modules/posixmodule.c | 103 ++++++++++-------- 2 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst new file mode 100644 index 00000000000..32f4e39f6d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst @@ -0,0 +1,5 @@ +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6390f1fc5fe..fc609b2707c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8431,53 +8431,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This should only be called from the parent process after +// This MUST only be called from the parent process after // PyOS_AfterFork_Parent(). static int -warn_about_fork_with_threads(const char* name) +warn_about_fork_with_threads( + const char* name, // Name of the API to use in the warning message. + const Py_ssize_t num_os_threads // Only trusted when >= 1. +) { // It's not safe to issue the warning while the world is stopped, because // other threads might be holding locks that we need, which would deadlock. assert(!_PyRuntime.stoptheworld.world_stopped); - // TODO: Consider making an `os` module API to return the current number - // of threads in the process. That'd presumably use this platform code but - // raise an error rather than using the inaccurate fallback. - Py_ssize_t num_python_threads = 0; -#if defined(__APPLE__) && defined(HAVE_GETPID) - mach_port_t macos_self = mach_task_self(); - mach_port_t macos_task; - if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { - thread_array_t macos_threads; - mach_msg_type_number_t macos_n_threads; - if (task_threads(macos_task, &macos_threads, - &macos_n_threads) == KERN_SUCCESS) { - num_python_threads = macos_n_threads; - } - } -#elif defined(__linux__) - // Linux /proc/self/stat 20th field is the number of threads. - FILE* proc_stat = fopen("/proc/self/stat", "r"); - if (proc_stat) { - size_t n; - // Size chosen arbitrarily. ~60% more bytes than a 20th column index - // observed on the author's workstation. - char stat_line[160]; - n = fread(&stat_line, 1, 159, proc_stat); - stat_line[n] = '\0'; - fclose(proc_stat); - - char *saveptr = NULL; - char *field = strtok_r(stat_line, " ", &saveptr); - unsigned int idx; - for (idx = 19; idx && field; --idx) { - field = strtok_r(NULL, " ", &saveptr); - } - if (idx == 0 && field) { // found the 20th field - num_python_threads = atoi(field); // 0 on error - } - } -#endif + Py_ssize_t num_python_threads = num_os_threads; if (num_python_threads <= 0) { // Fall back to just the number our threading module knows about. // An incomplete view of the world, but better than nothing. @@ -8530,6 +8496,51 @@ warn_about_fork_with_threads(const char* name) } return 0; } + +// If this returns <= 0, we were unable to successfully use any OS APIs. +// Returns a positive number of threads otherwise. +static Py_ssize_t get_number_of_os_threads(void) +{ + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + return num_python_threads; +} #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 @@ -8564,10 +8575,12 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork1") < 0) { + if (warn_about_fork_with_threads("fork1", num_os_threads) < 0) { return NULL; } } @@ -8615,10 +8628,12 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork") < 0) + if (warn_about_fork_with_threads("fork", num_os_threads) < 0) return NULL; } if (pid == -1) { @@ -9476,6 +9491,8 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); /* set O_CLOEXEC on master_fd */ @@ -9485,7 +9502,7 @@ os_forkpty_impl(PyObject *module) } // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("forkpty") < 0) + if (warn_about_fork_with_threads("forkpty", num_os_threads) < 0) return NULL; } if (pid == -1) { From 63548b36998e7f7cd5c7c28b53b348a93f836737 Mon Sep 17 00:00:00 2001 From: Shamil Date: Thu, 13 Nov 2025 14:01:31 +0300 Subject: [PATCH 168/417] gh-140260: fix data race in `_struct` module initialization with subinterpreters (#140909) --- Lib/test/test_struct.py | 17 ++++ ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 + Modules/_struct.c | 91 ++++++++++--------- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 75c76a36ee9..cceecdd526c 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -800,6 +800,23 @@ def test_c_complex_round_trip(self): round_trip = struct.unpack(f, struct.pack(f, z))[0] self.assertComplexesAreIdentical(z, round_trip) + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst new file mode 100644 index 00000000000..96bf9b51e48 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst @@ -0,0 +1,2 @@ +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. diff --git a/Modules/_struct.c b/Modules/_struct.c index f09252e82c3..2acb3df3a30 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -1505,6 +1506,53 @@ static formatdef lilendian_table[] = { {0} }; +/* Ensure endian table optimization happens exactly once across all interpreters */ +static _PyOnceFlag endian_tables_init_once = {0}; + +static int +init_endian_tables(void *Py_UNUSED(arg)) +{ + const formatdef *native = native_table; + formatdef *other, *ptr; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif + /* Scan through the native table, find a matching + entry in the endian table and swap in the + native implementations whenever possible + (64-bit platforms may not have "standard" sizes) */ + while (native->format != '\0' && other->format != '\0') { + ptr = other; + while (ptr->format != '\0') { + if (ptr->format == native->format) { + /* Match faster when formats are + listed in the same order */ + if (ptr == other) + other++; + /* Only use the trick if the + size matches */ + if (ptr->size != native->size) + break; + /* Skip float and double, could be + "unknown" float format */ + if (ptr->format == 'd' || ptr->format == 'f') + break; + /* Skip _Bool, semantics are different for standard size */ + if (ptr->format == '?') + break; + ptr->pack = native->pack; + ptr->unpack = native->unpack; + break; + } + ptr++; + } + native++; + } + return 0; +} + static const formatdef * whichtable(const char **pfmt) @@ -2710,47 +2758,8 @@ _structmodule_exec(PyObject *m) return -1; } - /* Check endian and swap in faster functions */ - { - const formatdef *native = native_table; - formatdef *other, *ptr; -#if PY_LITTLE_ENDIAN - other = lilendian_table; -#else - other = bigendian_table; -#endif - /* Scan through the native table, find a matching - entry in the endian table and swap in the - native implementations whenever possible - (64-bit platforms may not have "standard" sizes) */ - while (native->format != '\0' && other->format != '\0') { - ptr = other; - while (ptr->format != '\0') { - if (ptr->format == native->format) { - /* Match faster when formats are - listed in the same order */ - if (ptr == other) - other++; - /* Only use the trick if the - size matches */ - if (ptr->size != native->size) - break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; - /* Skip _Bool, semantics are different for standard size */ - if (ptr->format == '?') - break; - ptr->pack = native->pack; - ptr->unpack = native->unpack; - break; - } - ptr++; - } - native++; - } - } + /* init cannot fail */ + (void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL); /* Add some symbolic constants to the module */ state->StructError = PyErr_NewException("struct.error", NULL, NULL); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 11a3cd794ff..4621ad250f4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -24,6 +24,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - +Modules/_struct.c - endian_tables_init_once - ##----------------------- ## other values (not Python-specific) From d8e6bdc0d083f4e76ac49574544555ad91257592 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:21:32 +0200 Subject: [PATCH 169/417] gh-135801: Add the module parameter to compile() etc (GH-139652) Many functions related to compiling or parsing Python code, such as compile(), ast.parse(), symtable.symtable(), and importlib.abc.InspectLoader.source_to_code() now allow to pass the module name used when filtering syntax warnings. --- Doc/library/ast.rst | 7 ++- Doc/library/functions.rst | 11 +++- Doc/library/importlib.rst | 10 +++- Doc/library/symtable.rst | 8 ++- Doc/whatsnew/3.15.rst | 7 +++ Include/internal/pycore_compile.h | 9 ++- Include/internal/pycore_parser.h | 3 +- Include/internal/pycore_pyerrors.h | 3 +- Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_symtable.h | 3 +- Lib/ast.py | 5 +- Lib/importlib/_bootstrap_external.py | 9 +-- Lib/importlib/abc.py | 11 ++-- Lib/modulefinder.py | 2 +- Lib/profiling/sampling/_sync_coordinator.py | 2 +- Lib/profiling/tracing/__init__.py | 2 +- Lib/runpy.py | 6 +- Lib/symtable.py | 4 +- Lib/test/test_ast/test_ast.py | 10 ++++ Lib/test/test_builtin.py | 3 +- Lib/test/test_cmd_line_script.py | 23 ++++++-- Lib/test/test_compile.py | 10 ++++ Lib/test/test_import/__init__.py | 15 +---- Lib/test/test_runpy.py | 43 ++++++++++++++ Lib/test/test_symtable.py | 10 ++++ Lib/test/test_zipimport_support.py | 23 ++++++++ Lib/zipimport.py | 6 +- ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 ++ Modules/clinic/symtablemodule.c.h | 58 ++++++++++++++++--- Modules/symtablemodule.c | 19 +++++- Parser/lexer/state.c | 2 + Parser/lexer/state.h | 1 + Parser/peg_api.c | 6 +- Parser/pegen.c | 8 ++- Parser/pegen.h | 2 +- Parser/string_parser.c | 2 +- Parser/tokenizer/helpers.c | 4 +- Programs/_freeze_module.py | 2 +- Programs/freeze_test_frozenmain.py | 2 +- Python/ast_preprocess.c | 8 ++- Python/bltinmodule.c | 25 ++++++-- Python/clinic/bltinmodule.c.h | 26 ++++++--- Python/compile.c | 30 ++++++---- Python/errors.c | 7 ++- Python/pythonrun.c | 38 ++++++++++-- Python/symtable.c | 4 +- .../peg_extension/peg_extension.c | 4 +- 47 files changed, 390 insertions(+), 115 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 49462167217..0ea3c3c59a6 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2205,10 +2205,10 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2261,6 +2261,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: next + Added the *module* parameter. + .. function:: unparse(ast_obj) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e9879397555..3257daf89d3 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -292,7 +292,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -334,6 +336,10 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`ValueError` if the source contains null bytes. @@ -371,6 +377,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: next + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 602a7100a12..03ba23b6216 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -459,7 +459,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -471,11 +471,19 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: next + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd6..c0d9e79197d 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -21,11 +21,17 @@ tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: next + Added the *module* parameter. Examining Symbol Tables diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c6089f63dee..3cb766978a7 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,13 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass + the module name. It is needed to unambiguous :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + New modules =========== diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1c60834fa20..527141b54d0 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -50,7 +52,8 @@ extern int _PyAST_Preprocess( int optimize, int ff_features, int syntax_check_only, - int enable_warnings); + int enable_warnings, + PyObject *module); typedef struct { diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dc..2c46f59ab7d 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -48,7 +48,8 @@ extern struct _mod* _PyParser_ASTFromString( PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e12..f80808fcc8c 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -123,7 +123,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index c2832098ddb..f954f1b63ef 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,6 +33,12 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497..9dbfa913219 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -188,7 +188,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Lib/ast.py b/Lib/ast.py index 983ac1710d0..d9743ba7ab4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -24,7 +24,7 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 035ae0fcae1..4ab0e79ea6e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -819,13 +819,14 @@ def get_source(self, fullname): name=fullname) from exc return decode_source(source_bytes) - def source_to_code(self, data, path, *, _optimize=-1): + def source_to_code(self, data, path, fullname=None, *, _optimize=-1): """Return the code object compiled from source. The 'data' argument can be any object type that compile() supports. """ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) + dont_inherit=True, optimize=_optimize, + module=fullname) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -894,7 +895,7 @@ def get_code(self, fullname): source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) + code_object = self.source_to_code(source_bytes, source_path, fullname) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): @@ -1186,7 +1187,7 @@ def get_source(self, fullname): return '' def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + return compile('', '', 'exec', dont_inherit=True, module=fullname) def create_module(self, spec): """Use default semantics for module creation.""" diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 1e47495f65f..5c13432b5bd 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -108,7 +108,7 @@ def get_code(self, fullname): source = self.get_source(fullname) if source is None: return None - return self.source_to_code(source) + return self.source_to_code(source, '', fullname) @abc.abstractmethod def get_source(self, fullname): @@ -120,12 +120,12 @@ def get_source(self, fullname): raise ImportError @staticmethod - def source_to_code(data, path=''): + def source_to_code(data, path='', fullname=None): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) + return compile(data, path, 'exec', dont_inherit=True, module=fullname) exec_module = _bootstrap_external._LoaderBasics.exec_module load_module = _bootstrap_external._LoaderBasics.load_module @@ -163,9 +163,8 @@ def get_code(self, fullname): try: path = self.get_filename(fullname) except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) + path = '' + return self.source_to_code(source, path, fullname) _register( ExecutionLoader, diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index ac478ee7f51..b115d99ab30 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -334,7 +334,7 @@ def load_module(self, fqname, fp, pathname, file_info): self.msgout(2, "load_module ->", m) return m if type == _PY_SOURCE: - co = compile(fp.read(), pathname, 'exec') + co = compile(fp.read(), pathname, 'exec', module=fqname) elif type == _PY_COMPILED: try: data = fp.read() diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 8716e654104..adb040e89cc 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -182,7 +182,7 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: # Compile and execute the script - code = compile(source_code, script_path, 'exec') + code = compile(source_code, script_path, 'exec', module='__main__') exec(code, {'__name__': '__main__', '__file__': script_path}) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e diff --git a/Lib/profiling/tracing/__init__.py b/Lib/profiling/tracing/__init__.py index 2dc7ea92c8c..a6b8edf7216 100644 --- a/Lib/profiling/tracing/__init__.py +++ b/Lib/profiling/tracing/__init__.py @@ -185,7 +185,7 @@ def main(): progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') + code = compile(fp.read(), progname, 'exec', module='__main__') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) module = importlib.util.module_from_spec(spec) diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282ee..f072498f6cb 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,7 +247,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -256,7 +256,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -283,7 +283,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/symtable.py b/Lib/symtable.py index 77475c3ffd9..4c832e68f94 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -17,13 +17,13 @@ __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] -def symtable(code, filename, compile_type): +def symtable(code, filename, compile_type, *, module=None): """ Return the toplevel *SymbolTable* for the source code. *filename* is the name of the file with the code and *compile_type* is the *compile()* mode argument. """ - top = _symtable.symtable(code, filename, compile_type) + top = _symtable.symtable(code, filename, compile_type, module=module) return _newSymbolTable(top, filename) class SymbolTableFactory: diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 551de5851da..fb4a441ca64 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1083,6 +1083,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, '') self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=r'') + ast.parse(source, filename, module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index fba46af6617..ce60a5d095d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1103,7 +1103,8 @@ def test_exec_filter_syntax_warnings_by_module(self): with warnings.catch_warnings(record=True) as wlog: warnings.simplefilter('error') - warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('always', module=r'package.module\z') + warnings.filterwarnings('error', module=r'') exec(source, {'__name__': 'package.module', '__file__': filename}) self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) for wm in wlog: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f8115cc8300..cc1a625a509 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -814,15 +814,26 @@ def test_filter_syntax_warnings_by_module(self): filename = support.findfile('test_import/data/syntax_warnings.py') rc, out, err = assert_python_ok( '-Werror', - '-Walways:::test.test_import.data.syntax_warnings', + '-Walways:::__main__', + '-Werror:::test.test_import.data.syntax_warnings', + '-Werror:::syntax_warnings', filename) self.assertEqual(err.count(b': SyntaxWarning: '), 6) - rc, out, err = assert_python_ok( - '-Werror', - '-Walways:::syntax_warnings', - filename) - self.assertEqual(err.count(b': SyntaxWarning: '), 6) + def test_zipfile_run_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with os_helper.temp_dir() as script_dir: + zip_name, _ = make_zip_pkg( + script_dir, 'test_zip', 'test_pkg', '__main__', source) + rc, out, err = assert_python_ok( + '-Werror', + '-Walways:::__main__', + '-Werror:::test_pkg.__main__', + os.path.join(zip_name, 'test_pkg') + ) + self.assertEqual(err.count(b': SyntaxWarning: '), 12) def tearDownModule(): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9c2364491fe..30f21875b22 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1759,6 +1759,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + compile(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @support.subTests('src', [ textwrap.dedent(""" def f(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index e87d8b7e7bb..fe669bb04df 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1259,20 +1259,7 @@ def test_filter_syntax_warnings_by_module(self): warnings.catch_warnings(record=True) as wlog): warnings.simplefilter('error') warnings.filterwarnings('always', module=module_re) - import test.test_import.data.syntax_warnings - self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) - filename = test.test_import.data.syntax_warnings.__file__ - for wm in wlog: - self.assertEqual(wm.filename, filename) - self.assertIs(wm.category, SyntaxWarning) - - module_re = r'syntax_warnings\z' - unload('test.test_import.data.syntax_warnings') - with (os_helper.temp_dir() as tmpdir, - temporary_pycache_prefix(tmpdir), - warnings.catch_warnings(record=True) as wlog): - warnings.simplefilter('error') - warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') import test.test_import.data.syntax_warnings self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) filename = test.test_import.data.syntax_warnings.__file__ diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index a2a07c04f58..cc76b72b963 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -763,6 +765,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2c00e04b8..094ab8f573e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -601,6 +601,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + symtable.symtable(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index ae8a8c99762..2b28f46149b 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -13,9 +13,12 @@ import inspect import linecache import unittest +import warnings +from test import support from test.support import os_helper from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) +from test.support import import_helper verbose = test.support.verbose @@ -236,6 +239,26 @@ def f(): # bdb/pdb applies normcase to its filename before displaying self.assertIn(os.path.normcase(run_name.encode('utf-8')), data) + def test_import_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with (os_helper.temp_dir() as tmpdir, + import_helper.DirsOnSysPath()): + zip_name, _ = make_zip_script(tmpdir, "test_zip", + filename, 'test_pkg/test_mod.py') + sys.path.insert(0, zip_name) + import_helper.unload('test_pkg.test_mod') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'test_pkg\.test_mod\z') + warnings.filterwarnings('error', module='test_mod') + import test_pkg.test_mod + self.assertEqual(sorted(wm.lineno for wm in wlog), + sorted([4, 7, 10, 13, 14, 21]*2)) + filename = test_pkg.test_mod.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + def tearDownModule(): test.support.reap_children() diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 340a7e07112..19279d1c2be 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -742,9 +742,9 @@ def _normalize_line_endings(source): # Given a string buffer containing Python source code, compile it # and return a code object. -def _compile_source(pathname, source): +def _compile_source(pathname, source, module): source = _normalize_line_endings(source) - return compile(source, pathname, 'exec', dont_inherit=True) + return compile(source, pathname, 'exec', dont_inherit=True, module=module) # Convert the date/time values found in the Zip archive to a value # that's compatible with the time stamp stored in .pyc files. @@ -815,7 +815,7 @@ def _get_module_code(self, fullname): except ImportError as exc: import_error = exc else: - code = _compile_source(modpath, data) + code = _compile_source(modpath, data, fullname) if code is None: # bad magic number or non-matching mtime # in byte code, try next diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst new file mode 100644 index 00000000000..96226a7c525 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst @@ -0,0 +1,6 @@ +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify +the module name. +It is needed to unambiguous :ref:`filter ` syntax warnings +by module name. diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index bd55d77c540..65352593f94 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,30 +2,67 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, source, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /, *, module=None)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); #define _SYMTABLE_SYMTABLE_METHODDEF \ - {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL, _symtable_symtable__doc__}, + {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL|METH_KEYWORDS, _symtable_symtable__doc__}, static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr); + PyObject *filename, const char *startstr, + PyObject *modname); static PyObject * -_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(module), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "module", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "symtable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; const char *startstr; + PyObject *modname = Py_None; - if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } source = args[0]; @@ -45,7 +82,12 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, source, filename, startstr); + if (!noptargs) { + goto skip_optional_kwonly; + } + modname = args[3]; +skip_optional_kwonly: + return_value = _symtable_symtable_impl(module, source, filename, startstr, modname); exit: /* Cleanup for filename */ @@ -53,4 +95,4 @@ exit: return return_value; } -/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0137be60c487c841 input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d353f406831..a24927a9db6 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -16,14 +16,17 @@ _symtable.symtable filename: unicode_fs_decoded startstr: str / + * + module as modname: object = None Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ + PyObject *filename, const char *startstr, + PyObject *modname) +/*[clinic end generated code: output=235ec5a87a9ce178 input=fbf9adaa33c7070d]*/ { struct symtable *st; PyObject *t; @@ -50,7 +53,17 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, Py_XDECREF(source_copy); return NULL; } - st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "symtable() argument 'module' must be str or None, not %T", + modname); + Py_XDECREF(source_copy); + return NULL; + } + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf, modname); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2de9004fe08..3663dc3eb7f 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -43,6 +43,7 @@ _PyTokenizer_tok_new(void) tok->encoding = NULL; tok->cont_line = 0; tok->filename = NULL; + tok->module = NULL; tok->decoding_readline = NULL; tok->decoding_buffer = NULL; tok->readline = NULL; @@ -91,6 +92,7 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->decoding_buffer); Py_XDECREF(tok->readline); Py_XDECREF(tok->filename); + Py_XDECREF(tok->module); if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { PyMem_Free(tok->buf); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 877127125a7..9cd196a114c 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -102,6 +102,7 @@ struct tok_state { int parenlinenostack[MAXLEVEL]; int parencolstack[MAXLEVEL]; PyObject *filename; + PyObject *module; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ diff --git a/Parser/peg_api.c b/Parser/peg_api.c index d4acc3e4935..e30ca0453bd 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -4,13 +4,15 @@ mod_ty _PyParser_ASTFromString(const char *str, PyObject* filename, int mode, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, + PyObject *module) { if (PySys_Audit("compile", "yO", str, filename) < 0) { return NULL; } - mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena); + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, + arena, module); return result; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 70493031656..a38e973b3f6 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1010,6 +1010,11 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; + tok->module = PyUnicode_FromString("__main__"); + if (tok->module == NULL) { + goto error; + } + int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, NULL, arena); @@ -1036,7 +1041,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, PyObject *module) { int exec_input = start_rule == Py_file_input; @@ -1054,6 +1059,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); + tok->module = Py_XNewRef(module); // We need to clear up from here on mod_ty result = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 6b49b3537a0..b8f887608b1 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -378,7 +378,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char const char *, const char *, PyCompilerFlags *, int *, PyObject **, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/string_parser.c b/Parser/string_parser.c index ebe68989d1a..b164dfbc81a 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -88,7 +88,7 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_in } if (PyErr_WarnExplicitObject(category, msg, p->tok->filename, - lineno, NULL, NULL) < 0) { + lineno, p->tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the Syntax/DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index e5e2eed2d34..a03531a7441 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -127,7 +127,7 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { Py_DECREF(msg); if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { @@ -166,7 +166,7 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char * } if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index ba638eef6c4..62274e4aa9c 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -23,7 +23,7 @@ def read_text(inpath: str) -> bytes: def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"" # exec == Py_file_input - code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + code = compile(text, filename, "exec", optimize=0, dont_inherit=True, module=name) return marshal.dumps(code) diff --git a/Programs/freeze_test_frozenmain.py b/Programs/freeze_test_frozenmain.py index 848fc31b3d6..1a986bbac2a 100644 --- a/Programs/freeze_test_frozenmain.py +++ b/Programs/freeze_test_frozenmain.py @@ -24,7 +24,7 @@ def dump(fp, filename, name): with tokenize.open(filename) as source_fp: source = source_fp.read() - code = compile(source, code_filename, 'exec') + code = compile(source, code_filename, 'exec', module=name) data = marshal.dumps(code) writecode(fp, name, data) diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index fe6fd9479d1..d45435257cc 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -16,6 +16,7 @@ typedef struct { typedef struct { PyObject *filename; + PyObject *module; int optimize; int ff_features; int syntax_check_only; @@ -71,7 +72,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState } int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, n->col_offset + 1, n->end_lineno, - n->end_col_offset + 1); + n->end_col_offset + 1, + state->module); Py_DECREF(msg); return ret < 0 ? 0 : 1; } @@ -969,11 +971,13 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTPreprocessState *st int _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features, int syntax_check_only, int enable_warnings) + int ff_features, int syntax_check_only, int enable_warnings, + PyObject *module) { _PyASTPreprocessState state; memset(&state, 0, sizeof(_PyASTPreprocessState)); state.filename = filename; + state.module = module; state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f6fadd936bb..c2d780ac9b9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -751,6 +751,7 @@ compile as builtin_compile dont_inherit: bool = False optimize: int = -1 * + module as modname: object = None _feature_version as feature_version: int = -1 Compile source into a code object that can be executed by exec() or eval(). @@ -770,8 +771,8 @@ in addition to any features explicitly specified. static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ + int optimize, PyObject *modname, int feature_version) +/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ { PyObject *source_copy; const char *str; @@ -800,6 +801,15 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, "compile(): invalid optimize value"); goto error; } + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "compile() argument 'module' must be str or None, not %T", + modname); + goto error; + } if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); @@ -845,8 +855,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, - arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, arena, + syntax_check_only, modname) < 0) + { _PyArena_Free(arena); goto error; } @@ -859,7 +870,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + &cf, optimize, arena, modname); } _PyArena_Free(arena); goto finally; @@ -877,7 +888,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + result = _Py_CompileStringObjectWithModule(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index adb82f45c25..f08e5847abe 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -238,7 +238,8 @@ PyDoc_STRVAR(builtin_chr__doc__, PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" -" dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" +" dont_inherit=False, optimize=-1, *, module=None,\n" +" _feature_version=-1)\n" "--\n" "\n" "Compile source into a code object that can be executed by exec() or eval().\n" @@ -260,7 +261,7 @@ PyDoc_STRVAR(builtin_compile__doc__, static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version); + int optimize, PyObject *modname, int feature_version); static PyObject * builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -268,7 +269,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 7 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -277,7 +278,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(_feature_version), }, + .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(module), &_Py_ID(_feature_version), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -286,14 +287,14 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "_feature_version", NULL}; + static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "module", "_feature_version", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compile", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[8]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; @@ -301,6 +302,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int flags = 0; int dont_inherit = 0; int optimize = -1; + PyObject *modname = Py_None; int feature_version = -1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, @@ -359,12 +361,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - feature_version = PyLong_AsInt(args[6]); + if (args[6]) { + modname = args[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + feature_version = PyLong_AsInt(args[7]); if (feature_version == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); + return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, modname, feature_version); exit: /* Cleanup for filename */ @@ -1277,4 +1285,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=7eada753dc2e046f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06500bcc9a341e68 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index e2f1c7e8eb5..6951c98500d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -104,11 +104,13 @@ typedef struct _PyCompiler { * (including instructions for nested code objects) */ int c_disable_warning; + PyObject *c_module; } compiler; static int compiler_setup(compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags *flags, int optimize, PyArena *arena) + PyCompilerFlags *flags, int optimize, PyArena *arena, + PyObject *module) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; @@ -126,6 +128,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } + c->c_module = Py_XNewRef(module); if (!flags) { flags = &local_flags; } @@ -136,7 +139,9 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0, 1)) { + if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, + 0, 1, module)) + { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -156,6 +161,7 @@ compiler_free(compiler *c) _PySymtable_Free(c->c_st); } Py_XDECREF(c->c_filename); + Py_XDECREF(c->c_module); Py_XDECREF(c->c_const_cache); Py_XDECREF(c->c_stack); PyMem_Free(c); @@ -163,13 +169,13 @@ compiler_free(compiler *c) static compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { compiler *c = PyMem_Calloc(1, sizeof(compiler)); if (c == NULL) { return NULL; } - if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { + if (compiler_setup(c, mod, filename, pflags, optimize, arena, module) < 0) { compiler_free(c); return NULL; } @@ -1221,7 +1227,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) return ERROR; } int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + loc.end_lineno, loc.end_col_offset + 1, + c->c_module); Py_DECREF(msg); return ret; } @@ -1476,10 +1483,10 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { assert(!PyErr_Occurred()); - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, module); if (c == NULL) { return NULL; } @@ -1492,7 +1499,8 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena, int no_const_folding) + int optimize, PyArena *arena, int no_const_folding, + PyObject *module) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1502,7 +1510,9 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding, 0)) { + if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, + no_const_folding, 0, module)) + { return -1; } return 0; @@ -1627,7 +1637,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, NULL); if (c == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Python/errors.c b/Python/errors.c index 9fe95cec0ab..5c6ac48371a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1960,10 +1960,11 @@ _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_o */ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset) + int end_lineno, int end_col_offset, + PyObject *module) { - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, - filename, lineno, NULL, NULL) < 0) + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, + module, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 45211e1b075..49ce0a97d47 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1252,12 +1252,19 @@ _PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, } else { name = &_Py_STR(anon_string); } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + goto done; + } - mod = _PyParser_ASTFromString(str, name, start, flags, arena); + mod = _PyParser_ASTFromString(str, name, start, flags, arena, module); + Py_XDECREF(module); - if (mod != NULL) { + if (mod != NULL) { ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); } + +done: Py_XDECREF(source); _PyArena_Free(arena); return ret; @@ -1407,8 +1414,17 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, return NULL; } } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + if (interactive_src) { + Py_DECREF(interactive_filename); + } + return NULL; + } - PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); + PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, + arena, module); + Py_XDECREF(module); if (co == NULL) { if (interactive_src) { Py_DECREF(interactive_filename); @@ -1507,6 +1523,14 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) +{ + return _Py_CompileStringObjectWithModule(str, filename, start, + flags, optimize, NULL); +} + +PyObject * +_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { PyCodeObject *co; mod_ty mod; @@ -1514,14 +1538,16 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, + syntax_check_only, module) < 0) + { _PyArena_Free(arena); return NULL; } @@ -1529,7 +1555,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, _PyArena_Free(arena); return result; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena); + co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); _PyArena_Free(arena); return (PyObject *)co; } diff --git a/Python/symtable.c b/Python/symtable.c index bcd7365f8e1..29cf9190a4e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -3137,7 +3137,7 @@ symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_Source struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, - int start, PyCompilerFlags *flags) + int start, PyCompilerFlags *flags, PyObject *module) { struct symtable *st; mod_ty mod; @@ -3147,7 +3147,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index 1587d53d594..2fec5b05129 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -8,7 +8,7 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar PyObject *result = NULL; if (mode == 2) { - result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena); + result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena, NULL); } else if (mode == 1) { result = PyAST_mod2obj(module); } else { @@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - &flags, arena); + &flags, arena, NULL); if (res == NULL) { goto error; } From 2fbd39666663cb5ca1c0e3021ce2e7bc72331020 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:37:01 +0200 Subject: [PATCH 170/417] gh-140601: Refactor ElementTree.iterparse() tests (GH-141499) Split existing tests on smaller methods and move them to separate class. Rename variable "content" to "it". Use BytesIO instead of StringIO. Add few more tests. --- Lib/test/test_xml_etree.py | 430 ++++++++++++++++++++----------------- 1 file changed, 228 insertions(+), 202 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index f65baa0cfae..25c084c8b9c 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -574,208 +574,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - self.assertIsNone(context.root) - action, elem = next(context) - self.assertIsNone(context.root) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - with open(SIMPLE_XMLFILE, 'rb') as source: - context = iterparse(source) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r""), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"\n" - b"text\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - it.close() - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - it.close() - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - - def test_iterparse_close(self): - iterparse = ET.iterparse - - it = iterparse(SIMPLE_XMLFILE) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - list(it) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - list(it) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1499,6 +1297,234 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br""), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + def test_non_utf8(self): + source = io.BytesIO( + b"\n" + b"text\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + def test_parsing_error(self): + source = io.BytesIO(b"junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + del it + gc_collect() + + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): def _feed(self, parser, data, chunk_size=None, flush=False): From 732224e1139f7ed4fe0259a2dad900f84910949e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 13 Nov 2025 05:19:44 -0800 Subject: [PATCH 171/417] gh-139871: Add `bytearray.take_bytes([n])` to efficiently extract `bytes` (GH-140128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update `bytearray` to contain a `bytes` and provide a zero-copy path to "extract" the `bytes`. This allows making several code paths more efficient. This does not move any codepaths to make use of this new API. The documentation changes include common code patterns which can be made more efficient with this API. --- When just changing `bytearray` to contain `bytes` I ran pyperformance on a `--with-lto --enable-optimizations --with-static-libpython` build and don't see any major speedups or slowdowns with this; all seems to be in the noise of my machine (Generally changes under 5% or benchmarks that don't touch bytes/bytearray). Co-authored-by: Victor Stinner Co-authored-by: Maurycy Pawłowski-Wieroński <5383+maurycy@users.noreply.github.com> --- Doc/library/stdtypes.rst | 24 ++ Doc/whatsnew/3.15.rst | 80 ++++++ Include/cpython/bytearrayobject.h | 16 +- Include/internal/pycore_bytesobject.h | 8 + Lib/test/test_bytes.py | 81 ++++++ Lib/test/test_capi/test_bytearray.py | 5 +- Lib/test/test_sys.py | 2 +- ...-10-14-18-24-16.gh-issue-139871.SWtuUz.rst | 2 + Objects/bytearrayobject.c | 238 ++++++++++++------ Objects/bytesobject.c | 8 +- Objects/clinic/bytearrayobject.c.h | 39 ++- 11 files changed, 407 insertions(+), 96 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 97e7e08364e..c539345e598 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3173,6 +3173,30 @@ objects. .. versionadded:: 3.14 + .. method:: take_bytes(n=None, /) + + Remove the first *n* bytes from the bytearray and return them as an immutable + :class:`bytes`. + By default (if *n* is ``None``), return all bytes and clear the bytearray. + + If *n* is negative, index from the end and take the first :func:`len` + plus *n* bytes. If *n* is out of bounds, raise :exc:`IndexError`. + + Taking less than the full length will leave remaining bytes in the + :class:`bytearray`, which requires a copy. If the remaining bytes should be + discarded, use :func:`~bytearray.resize` or :keyword:`del` to truncate + then :func:`~bytearray.take_bytes` without a size. + + .. impl-detail:: + + Taking all bytes is a zero-copy operation. + + .. versionadded:: next + + See the :ref:`What's New ` entry for + common code patterns which can be optimized with + :func:`bytearray.take_bytes`. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 3cb766978a7..d7c9a41eeb2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,86 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +.. _whatsnew315-bytearray-take-bytes: + +* Added :meth:`bytearray.take_bytes(n=None, /) ` to take + bytes out of a :class:`bytearray` without copying. This enables optimizing code + which must return :class:`bytes` after working with a mutable buffer of bytes + such as data buffering, network protocol parsing, encoding, decoding, + and compression. Common code patterns which can be optimized with + :func:`~bytearray.take_bytes` are listed below. + + (Contributed by Cody Maloney in :gh:`139871`.) + + .. list-table:: Suggested Optimizing Refactors + :header-rows: 1 + + * - Description + - Old + - New + + * - Return :class:`bytes` after working with :class:`bytearray` + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return bytes(buffer) + + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return buffer.take_bytes() + + * - Empty a buffer getting the bytes + - .. code:: python + + buffer = bytearray(1024) + ... + data = bytes(buffer) + buffer.clear() + + - .. code:: python + + buffer = bytearray(1024) + ... + data = buffer.take_bytes() + + * - Split a buffer at a specific separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n + 1]) + del buffer[:n + 1] + assert data == b'abc' + assert buffer == bytearray(b'def') + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = buffer.take_bytes(n + 1) + + * - Split a buffer at a specific separator; discard after the separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n]) + buffer.clear() + assert data == b'abc' + assert len(buffer) == 0 + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + buffer.resize(n) + data = buffer.take_bytes() + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h index 4dddef713ce..1edd0820742 100644 --- a/Include/cpython/bytearrayobject.h +++ b/Include/cpython/bytearrayobject.h @@ -5,25 +5,25 @@ /* Object layout */ typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + /* How many bytes allocated in ob_bytes + + In the current implementation this is equivalent to Py_SIZE(ob_bytes_object). + The value is always loaded and stored atomically for thread safety. + There are API compatibilty concerns with removing so keeping for now. */ + Py_ssize_t ob_alloc; char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ Py_ssize_t ob_exports; /* How many buffer exports */ + PyObject *ob_bytes_object; /* PyBytes for zero-copy bytes conversion */ } PyByteArrayObject; -PyAPI_DATA(char) _PyByteArray_empty_string[]; - /* Macros and static inline functions, trading safety for speed */ #define _PyByteArray_CAST(op) \ (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op)) static inline char* PyByteArray_AS_STRING(PyObject *op) { - PyByteArrayObject *self = _PyByteArray_CAST(op); - if (Py_SIZE(self)) { - return self->ob_start; - } - return _PyByteArray_empty_string; + return _PyByteArray_CAST(op)->ob_start; } #define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self)) diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index c7bc53b6073..8e8fa696ee0 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -60,6 +60,14 @@ PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +/* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation + for a bytes object of length n should request PyBytesObject_SIZE + n bytes. + + Using _PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves + 3 or 7 bytes per bytes object allocation on a typical system. +*/ +#define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) + /* --- PyBytesWriter ------------------------------------------------------ */ struct PyBytesWriter { diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e012042159d..86898bfcab9 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1397,6 +1397,16 @@ def test_clear(self): b.append(ord('p')) self.assertEqual(b, b'p') + # Cleared object should be empty. + b = bytearray(b'abc') + b.clear() + self.assertEqual(b.__alloc__(), 0) + base_size = sys.getsizeof(bytearray()) + self.assertEqual(sys.getsizeof(b), base_size) + c = b.copy() + self.assertEqual(c.__alloc__(), 0) + self.assertEqual(sys.getsizeof(c), base_size) + def test_copy(self): b = bytearray(b'abc') bb = b.copy() @@ -1458,6 +1468,61 @@ def test_resize(self): self.assertRaises(MemoryError, bytearray().resize, sys.maxsize) self.assertRaises(MemoryError, bytearray(1000).resize, sys.maxsize) + def test_take_bytes(self): + ba = bytearray(b'ab') + self.assertEqual(ba.take_bytes(), b'ab') + self.assertEqual(len(ba), 0) + self.assertEqual(ba, bytearray(b'')) + self.assertEqual(ba.__alloc__(), 0) + base_size = sys.getsizeof(bytearray()) + self.assertEqual(sys.getsizeof(ba), base_size) + + # Positive and negative slicing. + ba = bytearray(b'abcdef') + self.assertEqual(ba.take_bytes(1), b'a') + self.assertEqual(ba, bytearray(b'bcdef')) + self.assertEqual(len(ba), 5) + self.assertEqual(ba.take_bytes(-5), b'') + self.assertEqual(ba, bytearray(b'bcdef')) + self.assertEqual(len(ba), 5) + self.assertEqual(ba.take_bytes(-3), b'bc') + self.assertEqual(ba, bytearray(b'def')) + self.assertEqual(len(ba), 3) + self.assertEqual(ba.take_bytes(3), b'def') + self.assertEqual(ba, bytearray(b'')) + self.assertEqual(len(ba), 0) + + # Take nothing from emptiness. + self.assertEqual(ba.take_bytes(0), b'') + self.assertEqual(ba.take_bytes(), b'') + self.assertEqual(ba.take_bytes(None), b'') + + # Out of bounds, bad take value. + self.assertRaises(IndexError, ba.take_bytes, -1) + self.assertRaises(TypeError, ba.take_bytes, 3.14) + ba = bytearray(b'abcdef') + self.assertRaises(IndexError, ba.take_bytes, 7) + + # Offset between physical and logical start (ob_bytes != ob_start). + ba = bytearray(b'abcde') + del ba[:2] + self.assertEqual(ba, bytearray(b'cde')) + self.assertEqual(ba.take_bytes(), b'cde') + + # Overallocation at end. + ba = bytearray(b'abcde') + del ba[-2:] + self.assertEqual(ba, bytearray(b'abc')) + self.assertEqual(ba.take_bytes(), b'abc') + ba = bytearray(b'abcde') + ba.resize(4) + self.assertEqual(ba.take_bytes(), b'abcd') + + # Take of a bytearray with references should fail. + ba = bytearray(b'abc') + with memoryview(ba) as mv: + self.assertRaises(BufferError, ba.take_bytes) + self.assertEqual(ba.take_bytes(), b'abc') def test_setitem(self): def setitem_as_mapping(b, i, val): @@ -2564,6 +2629,18 @@ def zfill(b, a): c = a.zfill(0x400000) assert not c or c[-1] not in (0xdd, 0xcd) + def take_bytes(b, a): # MODIFIES! + b.wait() + c = a.take_bytes() + assert not c or c[0] == 48 # '0' + + def take_bytes_n(b, a): # MODIFIES! + b.wait() + try: + c = a.take_bytes(10) + assert c == b'0123456789' + except IndexError: pass + def check(funcs, a=None, *args): if a is None: a = bytearray(b'0' * 0x400000) @@ -2625,6 +2702,10 @@ def check(funcs, a=None, *args): check([clear] + [startswith] * 10) check([clear] + [strip] * 10) + check([clear] + [take_bytes] * 10) + check([take_bytes_n] * 10, bytearray(b'0123456789' * 0x400)) + check([take_bytes_n] * 10, bytearray(b'0123456789' * 5)) + check([clear] + [contains] * 10) check([clear] + [subscript] * 10) check([clear2] + [ass_subscript2] * 10, None, bytearray(b'0' * 0x400000)) diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 52565ea34c6..cb7ad8b2225 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,3 +1,4 @@ +import sys import unittest from test.support import import_helper @@ -55,7 +56,9 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'', 0), bytearray()) self.assertEqual(fromstringandsize(NULL, 0), bytearray()) self.assertEqual(len(fromstringandsize(NULL, 3)), 3) - self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, + PY_SSIZE_T_MAX-sys.getsizeof(b'') + 1) self.assertRaises(SystemError, fromstringandsize, b'abc', -1) self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3ceed019ac4..9d3248d972e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1583,7 +1583,7 @@ def test_objecttypes(self): samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, vsize('n2Pi') + x.__alloc__()) + check(x, vsize('n2PiP') + x.__alloc__()) # bytearray_iterator check(iter(bytearray()), size('nP')) # bytes diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst new file mode 100644 index 00000000000..d4b8578afe3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst @@ -0,0 +1,2 @@ +Update :class:`bytearray` to use a :class:`bytes` under the hood as its buffer +and add :func:`bytearray.take_bytes` to take it out. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a73bfff340c..99bfdec89f6 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -17,8 +17,8 @@ class bytearray "PyByteArrayObject *" "&PyByteArray_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=5535b77c37a119e0]*/ -/* For PyByteArray_AS_STRING(). */ -char _PyByteArray_empty_string[] = ""; +/* Max number of bytes a bytearray can contain */ +#define PyByteArray_SIZE_MAX ((Py_ssize_t)(PY_SSIZE_T_MAX - _PyBytesObject_SIZE)) /* Helpers */ @@ -43,6 +43,14 @@ _getbytevalue(PyObject* arg, int *value) return 1; } +static void +bytearray_reinit_from_bytes(PyByteArrayObject *self, Py_ssize_t size, + Py_ssize_t alloc) { + self->ob_bytes = self->ob_start = PyBytes_AS_STRING(self->ob_bytes_object); + Py_SET_SIZE(self, size); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->ob_alloc, alloc); +} + static int bytearray_getbuffer_lock_held(PyObject *self, Py_buffer *view, int flags) { @@ -127,7 +135,6 @@ PyObject * PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) { PyByteArrayObject *new; - Py_ssize_t alloc; if (size < 0) { PyErr_SetString(PyExc_SystemError, @@ -135,35 +142,32 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) return NULL; } - /* Prevent buffer overflow when setting alloc to size+1. */ - if (size == PY_SSIZE_T_MAX) { - return PyErr_NoMemory(); - } - new = PyObject_New(PyByteArrayObject, &PyByteArray_Type); - if (new == NULL) + if (new == NULL) { return NULL; + } - if (size == 0) { - new->ob_bytes = NULL; - alloc = 0; - } - else { - alloc = size + 1; - new->ob_bytes = PyMem_Malloc(alloc); - if (new->ob_bytes == NULL) { - Py_DECREF(new); - return PyErr_NoMemory(); - } - if (bytes != NULL && size > 0) - memcpy(new->ob_bytes, bytes, size); - new->ob_bytes[size] = '\0'; /* Trailing null byte */ - } - Py_SET_SIZE(new, size); - new->ob_alloc = alloc; - new->ob_start = new->ob_bytes; + /* Fill values used in bytearray_dealloc. + + In an optimized build the memory isn't zeroed and ob_exports would be + uninitialized when when PyBytes_FromStringAndSize errored leading to + intermittent test failures. */ new->ob_exports = 0; + /* Optimization: size=0 bytearray should not allocate space + + PyBytes_FromStringAndSize returns the empty bytes global when size=0 so + no allocation occurs. */ + new->ob_bytes_object = PyBytes_FromStringAndSize(NULL, size); + if (new->ob_bytes_object == NULL) { + Py_DECREF(new); + return NULL; + } + bytearray_reinit_from_bytes(new, size, size); + if (bytes != NULL && size > 0) { + memcpy(new->ob_bytes, bytes, size); + } + return (PyObject *)new; } @@ -189,7 +193,6 @@ static int bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - void *sval; PyByteArrayObject *obj = ((PyByteArrayObject *)self); /* All computations are done unsigned to avoid integer overflows (see issue #22335). */ @@ -214,16 +217,17 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) return -1; } - if (size + logical_offset + 1 <= alloc) { + if (size + logical_offset <= alloc) { /* Current buffer is large enough to host the requested size, decide on a strategy. */ if (size < alloc / 2) { /* Major downsize; resize down to exact size */ - alloc = size + 1; + alloc = size; } else { /* Minor downsize; quick exit */ Py_SET_SIZE(self, size); + /* Add mid-buffer null; end provided by bytes. */ PyByteArray_AS_STRING(self)[size] = '\0'; /* Trailing null */ return 0; } @@ -236,38 +240,36 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) } else { /* Major upsize; resize up to exact size */ - alloc = size + 1; + alloc = size; } } - if (alloc > PY_SSIZE_T_MAX) { + if (alloc > PyByteArray_SIZE_MAX) { PyErr_NoMemory(); return -1; } + /* Re-align data to the start of the allocation. */ if (logical_offset > 0) { - sval = PyMem_Malloc(alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } - memcpy(sval, PyByteArray_AS_STRING(self), - Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyMem_Free(obj->ob_bytes); - } - else { - sval = PyMem_Realloc(obj->ob_bytes, alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } + /* optimization tradeoff: This is faster than a new allocation when + the number of bytes being removed in a resize is small; for large + size changes it may be better to just make a new bytes object as + _PyBytes_Resize will do a malloc + memcpy internally. */ + memmove(obj->ob_bytes, obj->ob_start, + Py_MIN(requested_size, Py_SIZE(self))); } - obj->ob_bytes = obj->ob_start = sval; - Py_SET_SIZE(self, size); - FT_ATOMIC_STORE_SSIZE_RELAXED(obj->ob_alloc, alloc); - obj->ob_bytes[size] = '\0'; /* Trailing null byte */ + int ret = _PyBytes_Resize(&obj->ob_bytes_object, alloc); + if (ret == -1) { + obj->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + size = alloc = 0; + } + bytearray_reinit_from_bytes(obj, size, alloc); + if (alloc != size) { + /* Add mid-buffer null; end provided by bytes. */ + obj->ob_bytes[size] = '\0'; + } - return 0; + return ret; } int @@ -295,7 +297,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) goto done; } - if (va.len > PY_SSIZE_T_MAX - vb.len) { + if (va.len > PyByteArray_SIZE_MAX - vb.len) { PyErr_NoMemory(); goto done; } @@ -339,7 +341,7 @@ bytearray_iconcat_lock_held(PyObject *op, PyObject *other) } Py_ssize_t size = Py_SIZE(self); - if (size > PY_SSIZE_T_MAX - vo.len) { + if (size > PyByteArray_SIZE_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } @@ -373,7 +375,7 @@ bytearray_repeat_lock_held(PyObject *op, Py_ssize_t count) count = 0; } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } Py_ssize_t size = mysize * count; @@ -409,7 +411,7 @@ bytearray_irepeat_lock_held(PyObject *op, Py_ssize_t count) } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } const Py_ssize_t size = mysize * count; @@ -585,7 +587,7 @@ bytearray_setslice_linear(PyByteArrayObject *self, buf = PyByteArray_AS_STRING(self); } else if (growth > 0) { - if (Py_SIZE(self) > (Py_ssize_t)PY_SSIZE_T_MAX - growth) { + if (Py_SIZE(self) > PyByteArray_SIZE_MAX - growth) { PyErr_NoMemory(); return -1; } @@ -899,6 +901,13 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, PyObject *it; PyObject *(*iternext)(PyObject *); + /* First __init__; set ob_bytes_object so ob_bytes is always non-null. */ + if (self->ob_bytes_object == NULL) { + self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + bytearray_reinit_from_bytes(self, 0, 0); + self->ob_exports = 0; + } + if (Py_SIZE(self) != 0) { /* Empty previous contents (yes, do this first of all!) */ if (PyByteArray_Resize((PyObject *)self, 0) < 0) @@ -1169,9 +1178,7 @@ bytearray_dealloc(PyObject *op) "deallocated bytearray object has exported buffers"); PyErr_Print(); } - if (self->ob_bytes != 0) { - PyMem_Free(self->ob_bytes); - } + Py_XDECREF(self->ob_bytes_object); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1491,6 +1498,82 @@ bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) } +/*[clinic input] +@critical_section +bytearray.take_bytes + n: object = None + Bytes to take, negative indexes from end. None indicates all bytes. + / +Take *n* bytes from the bytearray and return them as a bytes object. +[clinic start generated code]*/ + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) +/*[clinic end generated code: output=3147fbc0bbbe8d94 input=b15b5172cdc6deda]*/ +{ + Py_ssize_t to_take; + Py_ssize_t size = Py_SIZE(self); + if (Py_IsNone(n)) { + to_take = size; + } + // Integer index, from start (zero, positive) or end (negative). + else if (_PyIndex_Check(n)) { + to_take = PyNumber_AsSsize_t(n, PyExc_IndexError); + if (to_take == -1 && PyErr_Occurred()) { + return NULL; + } + if (to_take < 0) { + to_take += size; + } + } + else { + PyErr_SetString(PyExc_TypeError, "n must be an integer or None"); + return NULL; + } + + if (to_take < 0 || to_take > size) { + PyErr_Format(PyExc_IndexError, + "can't take %zd bytes outside size %zd", + to_take, size); + return NULL; + } + + // Exports may change the contents. No mutable bytes allowed. + if (!_canresize(self)) { + return NULL; + } + + if (to_take == 0 || size == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + // Copy remaining bytes to a new bytes. + Py_ssize_t remaining_length = size - to_take; + PyObject *remaining = PyBytes_FromStringAndSize(self->ob_start + to_take, + remaining_length); + if (remaining == NULL) { + return NULL; + } + + // If the bytes are offset inside the buffer must first align. + if (self->ob_start != self->ob_bytes) { + memmove(self->ob_bytes, self->ob_start, to_take); + self->ob_start = self->ob_bytes; + } + + if (_PyBytes_Resize(&self->ob_bytes_object, to_take) == -1) { + Py_DECREF(remaining); + return NULL; + } + + // Point the bytearray towards the buffer with the remaining data. + PyObject *result = self->ob_bytes_object; + self->ob_bytes_object = remaining; + bytearray_reinit_from_bytes(self, remaining_length, remaining_length); + return result; +} + + /*[clinic input] @critical_section bytearray.translate @@ -1868,11 +1951,6 @@ bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item) Py_ssize_t n = Py_SIZE(self); char *buf; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; buf = PyByteArray_AS_STRING(self); @@ -1987,11 +2065,6 @@ bytearray_append_impl(PyByteArrayObject *self, int item) { Py_ssize_t n = Py_SIZE(self); - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; @@ -2099,16 +2172,16 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) if (len >= buf_size) { Py_ssize_t addition; - if (len == PY_SSIZE_T_MAX) { + if (len == PyByteArray_SIZE_MAX) { Py_DECREF(it); Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } addition = len >> 1; - if (addition > PY_SSIZE_T_MAX - len - 1) - buf_size = PY_SSIZE_T_MAX; + if (addition > PyByteArray_SIZE_MAX - len) + buf_size = PyByteArray_SIZE_MAX; else - buf_size = len + addition + 1; + buf_size = len + addition; if (bytearray_resize_lock_held((PyObject *)bytearray_obj, buf_size) < 0) { Py_DECREF(it); Py_DECREF(bytearray_obj); @@ -2405,7 +2478,11 @@ static PyObject * bytearray_alloc(PyObject *op, PyObject *Py_UNUSED(ignored)) { PyByteArrayObject *self = _PyByteArray_CAST(op); - return PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + alloc += _PyBytesObject_SIZE; + } + return PyLong_FromSsize_t(alloc); } /*[clinic input] @@ -2601,9 +2678,13 @@ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self) /*[clinic end generated code: output=738abdd17951c427 input=e27320fd98a4bc5a]*/ { - size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc) * sizeof(char); - return PyLong_FromSize_t(res); + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(self)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + res += _PyBytesObject_SIZE + alloc; + } + + return PyLong_FromSsize_t(res); } static PySequenceMethods bytearray_as_sequence = { @@ -2686,6 +2767,7 @@ static PyMethodDef bytearray_methods[] = { BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", bytearray_swapcase, METH_NOARGS, _Py_swapcase__doc__}, + BYTEARRAY_TAKE_BYTES_METHODDEF {"title", bytearray_title, METH_NOARGS, _Py_title__doc__}, BYTEARRAY_TRANSLATE_METHODDEF {"upper", bytearray_upper, METH_NOARGS, _Py_upper__doc__}, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2b9513abe91..2b0925017f2 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -25,13 +25,7 @@ class bytes "PyBytesObject *" "&PyBytes_Type" #include "clinic/bytesobject.c.h" -/* PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation - for a bytes object of length n should request PyBytesObject_SIZE + n bytes. - - Using PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves - 3 or 7 bytes per bytes object allocation on a typical system. -*/ -#define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +#define PyBytesObject_SIZE _PyBytesObject_SIZE /* Forward declaration */ static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 6f13865177d..be704ccf68f 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -631,6 +631,43 @@ exit: return return_value; } +PyDoc_STRVAR(bytearray_take_bytes__doc__, +"take_bytes($self, n=None, /)\n" +"--\n" +"\n" +"Take *n* bytes from the bytearray and return them as a bytes object.\n" +"\n" +" n\n" +" Bytes to take, negative indexes from end. None indicates all bytes."); + +#define BYTEARRAY_TAKE_BYTES_METHODDEF \ + {"take_bytes", _PyCFunction_CAST(bytearray_take_bytes), METH_FASTCALL, bytearray_take_bytes__doc__}, + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n); + +static PyObject * +bytearray_take_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n = Py_None; + + if (!_PyArg_CheckPositional("take_bytes", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + n = args[0]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = bytearray_take_bytes_impl((PyByteArrayObject *)self, n); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -1796,4 +1833,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=fdfe41139c91e409 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5eddefde2a001ceb input=a9049054013a1b77]*/ From c2470b39fa21f355f811419f1b3d1c776c36fb10 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Thu, 13 Nov 2025 17:44:40 +0300 Subject: [PATCH 172/417] gh-137959: Fix `TIER1_TO_TIER2` macro name in JIT InternalDocs (GH-141496) JIT InternalDocs fix --- InternalDocs/jit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index 09585380737..1740b22b85f 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -53,7 +53,7 @@ ## The micro-op optimizer ## The JIT interpreter After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop -executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. +executor, it transfers control to this executor via the `TIER1_TO_TIER2` macro. CPython implements two executors. Here we describe the JIT interpreter, which is the simpler of them and is therefore useful for debugging and analyzing From f72768f30e6ed9253eb3b6374b4395dfcaf4842a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 13 Nov 2025 10:02:21 -0500 Subject: [PATCH 173/417] gh-141004: Document C APIs for dictionary keys, values, and items (GH-141009) Co-authored-by: Petr Viktorin --- Doc/c-api/dict.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 246ce5391e1..b7f201811aa 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -431,3 +431,49 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary View Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. From d7862e9b1bd8f82e41c4f2c4dad31e15707d856f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 13 Nov 2025 10:07:57 -0500 Subject: [PATCH 174/417] gh-141004: Document `PyCode_Optimize` (GH-141378) --- Doc/c-api/code.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index c9741b61254..45f5e83adc4 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -211,6 +211,17 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a :term:`soft deprecated` function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + .. _c_codeobject_flags: Code Object Flags From b99db92dde38b17c3fba3b5db76a383ceddfce49 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 17:30:50 +0100 Subject: [PATCH 175/417] gh-139653: Add PyUnstable_ThreadState_SetStackProtection() (#139668) Add PyUnstable_ThreadState_SetStackProtection() and PyUnstable_ThreadState_ResetStackProtection() functions to set the stack base address and stack size of a Python thread state. Co-authored-by: Petr Viktorin --- Doc/c-api/exceptions.rst | 3 + Doc/c-api/init.rst | 37 +++++++++ Doc/whatsnew/3.15.rst | 6 ++ Include/cpython/pystate.h | 12 +++ Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_tstate.h | 4 + ...-10-06-22-17-47.gh-issue-139653.6-1MOd.rst | 4 + Modules/_testinternalcapi.c | 54 +++++++++++++ Python/ceval.c | 77 +++++++++++++++++-- Python/pystate.c | 3 + 10 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 5241533e112..0ee595a07ac 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -976,6 +976,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 49ffeab5585..18ee1611807 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1366,6 +1366,43 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.11 +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: next + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: next + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index d7c9a41eeb2..b360ad964cf 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1066,6 +1066,12 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) + Changed C APIs -------------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index dd2ea1202b3..c53abe43ebe 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -276,6 +276,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index f954f1b63ef..04a557e1204 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -60,6 +60,12 @@ extern PyObject * _Py_CompileStringObjectWithModule( # define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) #endif +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + #ifdef __cplusplus } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 29ebdfd7e01..a44c523e202 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -37,6 +37,10 @@ typedef struct _PyThreadStateImpl { uintptr_t c_stack_soft_limit; uintptr_t c_stack_hard_limit; + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference diff --git a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst new file mode 100644 index 00000000000..cd3d5262fa0 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index dede05960d7..6514ca7f3cd 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2446,6 +2446,58 @@ module_get_gc_hooks(PyObject *self, PyObject *arg) return result; } + +static void +check_threadstate_set_stack_protection(PyThreadState *tstate, + void *start, size_t size) +{ + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == 0); + assert(!PyErr_Occurred()); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_top == (uintptr_t)start + size); + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +} + + +static PyObject * +test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_GET(); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(!PyErr_Occurred()); + + uintptr_t init_base = ts->c_stack_init_base; + size_t init_top = ts->c_stack_init_top; + + // Test the minimum stack size + size_t size = _PyOS_MIN_STACK_SIZE; + void *start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test a larger size + size = 7654321; + assert(size > _PyOS_MIN_STACK_SIZE); + start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test invalid size (too small) + size = 5; + start = (void*)(_Py_get_machine_stack_pointer() - size); + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == -1); + assert(PyErr_ExceptionMatches(PyExc_ValueError)); + PyErr_Clear(); + + // Test PyUnstable_ThreadState_ResetStackProtection() + PyUnstable_ThreadState_ResetStackProtection(tstate); + assert(ts->c_stack_init_base == init_base); + assert(ts->c_stack_init_top == init_top); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2556,6 +2608,8 @@ static PyMethodDef module_functions[] = { {"simple_pending_call", simple_pending_call, METH_O}, {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, + {"test_threadstate_set_stack_protection", + test_threadstate_set_stack_protection, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/ceval.c b/Python/ceval.c index 43e8ee71206..07d21575e3a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -443,7 +443,7 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif static void -hardware_stack_limits(uintptr_t *top, uintptr_t *base) +hardware_stack_limits(uintptr_t *base, uintptr_t *top) { #ifdef WIN32 ULONG_PTR low, high; @@ -486,23 +486,86 @@ hardware_stack_limits(uintptr_t *top, uintptr_t *base) #endif } -void -_Py_InitializeRecursionLimits(PyThreadState *tstate) +static void +tstate_set_stack(PyThreadState *tstate, + uintptr_t base, uintptr_t top) { - uintptr_t top; - uintptr_t base; - hardware_stack_limits(&top, &base); + assert(base < top); + assert((top - base) >= _PyOS_MIN_STACK_SIZE); + #ifdef _Py_THREAD_SANITIZER // Thread sanitizer crashes if we use more than half the stack. uintptr_t stacksize = top - base; - base += stacksize/2; + base += stacksize / 2; #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; _tstate->c_stack_top = top; _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; + +#ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +#endif } + +void +_Py_InitializeRecursionLimits(PyThreadState *tstate) +{ + uintptr_t base, top; + hardware_stack_limits(&base, &top); + assert(top != 0); + + tstate_set_stack(tstate, base, top); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + ts->c_stack_init_base = base; + ts->c_stack_init_top = top; + + // Test the stack pointer +#if !defined(NDEBUG) && !defined(__wasi__) + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + assert(ts->c_stack_soft_limit < here_addr); + assert(here_addr < ts->c_stack_top); +#endif +} + + +int +PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, + void *stack_start_addr, size_t stack_size) +{ + if (stack_size < _PyOS_MIN_STACK_SIZE) { + PyErr_Format(PyExc_ValueError, + "stack_size must be at least %zu bytes", + _PyOS_MIN_STACK_SIZE); + return -1; + } + + uintptr_t base = (uintptr_t)stack_start_addr; + uintptr_t top = base + stack_size; + tstate_set_stack(tstate, base, top); + return 0; +} + + +void +PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->c_stack_init_top != 0) { + tstate_set_stack(tstate, + ts->c_stack_init_base, + ts->c_stack_init_top); + return; + } + + _Py_InitializeRecursionLimits(tstate); +} + + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ int diff --git a/Python/pystate.c b/Python/pystate.c index cf251c120d7..341c680a403 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1495,6 +1495,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->c_stack_top = 0; _tstate->c_stack_hard_limit = 0; + _tstate->c_stack_init_base = 0; + _tstate->c_stack_init_top = 0; + _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; From b2b68d40f887c8a9583a9b48babc40f25bc5e0e2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 19:48:52 +0200 Subject: [PATCH 176/417] gh-140873: Add support of non-descriptor callables in functools.singledispatchmethod() (GH-140884) --- Doc/library/functools.rst | 5 ++- Doc/whatsnew/3.15.rst | 8 +++++ Lib/functools.py | 5 ++- Lib/test/test_functools.py | 35 ++++++++++++++++++- ...-11-01-14-44-09.gh-issue-140873.kfuc9B.rst | 2 ++ 5 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 1d9ac328f32..b2e2e11c0dc 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -672,7 +672,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -716,6 +716,9 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 + .. versionchanged:: next + Added support of non-:term:`descriptor` callables. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index b360ad964cf..895616e3049 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -498,6 +498,14 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +functools +--------- + +* :func:`~functools.singledispatchmethod` now supports non-:term:`descriptor` + callables. + (Contributed by Serhiy Storchaka in :gh:`140873`.) + + hashlib ------- diff --git a/Lib/functools.py b/Lib/functools.py index a92844ba722..8063eb5ffc3 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -1083,7 +1083,10 @@ def __call__(self, /, *args, **kwargs): 'singledispatchmethod method') raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return self._dispatch(args[0].__class__).__get__(self._obj, self._cls)(*args, **kwargs) + method = self._dispatch(args[0].__class__) + if hasattr(method, "__get__"): + method = method.__get__(self._obj, self._cls) + return method(*args, **kwargs) def __getattr__(self, name): # Resolve these attributes lazily to speed up creation of diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ce9e7f6d57d..090926fd8d8 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2785,7 +2785,7 @@ class Slot: @functools.singledispatchmethod @classmethod def go(cls, item, arg): - pass + return item - arg @go.register @classmethod @@ -2794,7 +2794,9 @@ def _(cls, item: int, arg): s = Slot() self.assertEqual(s.go(1, 1), 2) + self.assertEqual(s.go(1.5, 1), 0.5) self.assertEqual(Slot.go(1, 1), 2) + self.assertEqual(Slot.go(1.5, 1), 0.5) def test_staticmethod_slotted_class(self): class A: @@ -3485,6 +3487,37 @@ def _(item, arg: bytes) -> str: self.assertEqual(str(Signature.from_callable(A.static_func)), '(item, arg: int) -> str') + def test_method_non_descriptor(self): + class Callable: + def __init__(self, value): + self.value = value + def __call__(self, arg): + return self.value, arg + + class A: + t = functools.singledispatchmethod(Callable('general')) + t.register(int, Callable('special')) + + @functools.singledispatchmethod + def u(self, arg): + return 'general', arg + u.register(int, Callable('special')) + + v = functools.singledispatchmethod(Callable('general')) + @v.register(int) + def _(self, arg): + return 'special', arg + + a = A() + self.assertEqual(a.t(0), ('special', 0)) + self.assertEqual(a.t(2.5), ('general', 2.5)) + self.assertEqual(A.t(0), ('special', 0)) + self.assertEqual(A.t(2.5), ('general', 2.5)) + self.assertEqual(a.u(0), ('special', 0)) + self.assertEqual(a.u(2.5), ('general', 2.5)) + self.assertEqual(a.v(0), ('special', 0)) + self.assertEqual(a.v(2.5), ('general', 2.5)) + class CachedCostItem: _cost = 1 diff --git a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst new file mode 100644 index 00000000000..e1505764064 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst @@ -0,0 +1,2 @@ +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. From 196f1519cd2d8134d7643536f13f2b2844bea65d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 13 Nov 2025 17:58:47 +0000 Subject: [PATCH 177/417] gh-141004: Document `PyErr_RangedSyntaxLocationObject` (#141521) PyErr_RangedSyntaxLocationObject --- Doc/c-api/exceptions.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 0ee595a07ac..d7fe9e2c9ec 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string From 4fa80ce74c6d9f5159bdc5ec3596a194f0391e21 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 14 Nov 2025 02:08:32 +0800 Subject: [PATCH 178/417] gh-139109: A new tracing JIT compiler frontend for CPython (GH-140310) This PR changes the current JIT model from trace projection to trace recording. Benchmarking: better pyperformance (about 1.7% overall) geomean versus current https://raw.githubusercontent.com/facebookexperimental/free-threading-benchmarking/refs/heads/main/results/bm-20251108-3.15.0a1%2B-7e2bc1d-JIT/bm-20251108-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-7e2bc1d-vs-base.svg, 100% faster Richards on the most improved benchmark versus the current JIT. Slowdown of about 10-15% on the worst benchmark versus the current JIT. **Note: the fastest version isn't the one merged, as it relies on fixing bugs in the specializing interpreter, which is left to another PR**. The speedup in the merged version is about 1.1%. https://raw.githubusercontent.com/facebookexperimental/free-threading-benchmarking/refs/heads/main/results/bm-20251112-3.15.0a1%2B-f8a764a-JIT/bm-20251112-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-f8a764a-vs-base.svg Stats: 50% more uops executed, 30% more traces entered the last time we ran them. It also suggests our trace lengths for a real trace recording JIT are too short, as a lot of trace too long aborts https://github.com/facebookexperimental/free-threading-benchmarking/blob/main/results/bm-20251023-3.15.0a1%2B-eb73378-CLANG%2CJIT/bm-20251023-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-eb73378-pystats-vs-base.md . This new JIT frontend is already able to record/execute significantly more instructions than the previous JIT frontend. In this PR, we are now able to record through custom dunders, simple object creation, generators, etc. None of these were done by the old JIT frontend. Some custom dunders uops were discovered to be broken as part of this work gh-140277 The optimizer stack space check is disabled, as it's no longer valid to deal with underflow. Pros: * Ignoring the generated tracer code as it's automatically created, this is only additional 1k lines of code. The maintenance burden is handled by the DSL and code generator. * `optimizer.c` is now significantly simpler, as we don't have to do strange things to recover the bytecode from a trace. * The new JIT frontend is able to handle a lot more control-flow than the old one. * Tracing is very low overhead. We use the tail calling interpreter/computed goto interpreter to switch between tracing mode and non-tracing mode. I call this mechanism dual dispatch, as we have two dispatch tables dispatching to each other. Specialization is still enabled while tracing. * Better handling of polymorphism. We leverage the specializing interpreter for this. Cons: * (For now) requires tail calling interpreter or computed gotos. This means no Windows JIT for now :(. Not to fret, tail calling is coming soon to Windows though https://github.com/python/cpython/pull/139962 Design: * After each instruction, the `record_previous_inst` function/label is executed. This does as the name suggests. * The tracing interpreter lowers bytecode to uops directly so that it can obtain "fresh" values at the point of lowering. * The tracing version behaves nearly identical to the normal interpreter, in fact it even has specialization! This allows it to run without much of a slowdown when tracing. The actual cost of tracing is only a function call and writes to memory. * The tracing interpreter uses the specializing interpreter's deopt to naturally form the side exit chains. This allows it to side exit chain effectively, without repeating much code. We force a re-specializing when tracing a deopt. * The tracing interpreter can even handle goto errors/exceptions, but I chose to disable them for now as it's not tested. * Because we do not share interpreter dispatch, there is should be no significant slowdown to the original specializing interpreter on tailcall and computed got with JIT disabled. With JIT enabled, there might be a slowdown in the form of the JIT trying to trace. * Things that could have dynamic instruction pointer effects are guarded on. The guard deopts to a new instruction --- `_DYNAMIC_EXIT`. --- .github/workflows/jit.yml | 26 +- Include/cpython/pystats.h | 2 + Include/internal/pycore_backoff.h | 17 +- Include/internal/pycore_ceval.h | 2 + Include/internal/pycore_interp_structs.h | 4 +- Include/internal/pycore_opcode_metadata.h | 71 +- Include/internal/pycore_optimizer.h | 41 +- Include/internal/pycore_tstate.h | 39 +- Include/internal/pycore_uop.h | 12 +- Include/internal/pycore_uop_ids.h | 389 +++--- Include/internal/pycore_uop_metadata.h | 38 +- Lib/test/test_ast/test_ast.py | 4 +- Lib/test/test_capi/test_opt.py | 65 +- Lib/test/test_sys.py | 5 +- ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 + Modules/_testinternalcapi.c | 3 +- Objects/codeobject.c | 1 + Objects/frameobject.c | 6 +- Objects/funcobject.c | 6 +- Python/bytecodes.c | 194 ++- Python/ceval.c | 55 +- Python/ceval_macros.h | 67 +- Python/executor_cases.c.h | 139 ++- Python/generated_cases.c.h | 104 +- Python/instrumentation.c | 2 + Python/jit.c | 2 +- Python/opcode_targets.h | 526 +++++++- Python/optimizer.c | 1063 +++++++++-------- Python/optimizer_analysis.c | 54 +- Python/optimizer_bytecodes.c | 137 ++- Python/optimizer_cases.c.h | 153 ++- Python/optimizer_symbols.c | 44 +- Python/pystate.c | 27 +- Tools/c-analyzer/cpython/ignored.tsv | 1 + Tools/cases_generator/analyzer.py | 58 + Tools/cases_generator/generators_common.py | 17 +- .../opcode_metadata_generator.py | 4 +- Tools/cases_generator/target_generator.py | 26 +- Tools/cases_generator/tier2_generator.py | 54 +- .../cases_generator/uop_metadata_generator.py | 4 +- Tools/jit/template.c | 11 +- 41 files changed, 2409 insertions(+), 1065 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 62325250bd3..3349eb04242 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,9 +57,10 @@ jobs: fail-fast: false matrix: target: - - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc - - aarch64-pc-windows-msvc/msvc +# To re-enable later when we support these. +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -70,15 +71,16 @@ jobs: llvm: - 21 include: - - target: i686-pc-windows-msvc/msvc - architecture: Win32 - runner: windows-2022 - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2022 - - target: aarch64-pc-windows-msvc/msvc - architecture: ARM64 - runner: windows-11-arm +# To re-enable later when we support these. +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-2022 +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-2022 +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index d0a925a3055..1c94603c08b 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -150,6 +150,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 454c8dde031..71066f1bd9f 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -95,11 +95,24 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + // Initial JUMP_BACKWARD counter. // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly // re-specialized: -#define JUMP_BACKWARD_INITIAL_VALUE 4095 +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformance +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 #define JUMP_BACKWARD_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -112,7 +125,7 @@ initial_jump_backoff_counter(void) * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_VALUE 4000 #define SIDE_EXIT_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fe72a0123eb..33b9fd053f7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -392,6 +392,8 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e8cbe9d894e..9e4504479cd 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,8 +14,6 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState -#include "pycore_uop.h" // struct _PyUOpInstruction - #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -934,10 +932,10 @@ struct _is { PyObject *common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; - struct _PyUOpInstruction *jit_uop_buffer; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; + struct _PyExecutorObject *cold_dynamic_executor; int executor_deletion_list_remaining_capacity; size_t executor_creation_counter; _rare_events rare_events; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd..548627dc798 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1031,6 +1031,8 @@ enum InstructionFormat { #define HAS_ERROR_NO_POP_FLAG (4096) #define HAS_NO_SAVE_IP_FLAG (8192) #define HAS_PERIODIC_FLAG (16384) +#define HAS_UNPREDICTABLE_JUMP_FLAG (32768) +#define HAS_NEEDS_GUARD_IP_FLAG (65536) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1046,6 +1048,8 @@ enum InstructionFormat { #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1062,7 +1066,7 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; @@ -1077,7 +1081,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1094,22 +1098,22 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,8 +1121,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1143,7 +1147,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1151,11 +1155,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1164,13 +1168,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1183,8 +1187,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1197,7 +1201,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1205,7 +1209,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1253,10 +1257,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1292,7 +1296,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1406,6 +1410,9 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8ed5436eb68..653285a2c6b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -21,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -44,7 +36,9 @@ typedef struct { typedef struct _PyExitData { uint32_t target; - uint16_t index; + uint16_t index:14; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -94,9 +88,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); // This value is arbitrary and was not optimized. #define JIT_CLEANUP_THRESHOLD 1000 -#define TRACE_STACK_SIZE 5 - -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize( + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); @@ -130,7 +123,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) +#define MAX_ABSTRACT_FRAME_DEPTH (16) // The maximum number of side exits that we can take before requiring forward // progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this @@ -258,6 +251,7 @@ struct _Py_UOpsAbstractFrame { int stack_len; int locals_len; PyFunctionObject *func; + PyCodeObject *code; JitOptRef *stack_pointer; JitOptRef *stack; @@ -333,11 +327,11 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { @@ -346,6 +340,7 @@ static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) } extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); @@ -354,7 +349,9 @@ static inline int is_terminator(const _PyUOpInstruction *uop) int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP + opcode == _DEOPT || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT ); } @@ -365,6 +362,18 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); + +int +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, + int oparg); + +void _PyJit_FinalizeTracing(PyThreadState *tstate); + +void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index a44c523e202..50048801b2e 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -12,7 +12,8 @@ extern "C" { #include "pycore_freelist_state.h" // struct _Py_freelists #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +22,38 @@ struct _gc_thread_state { }; #endif +#if _Py_TIER2 +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +typedef struct _PyJitTracerPreviousState { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerState { + _PyUOpInstruction *code_buffer; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; +} _PyJitTracerState; +#endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -85,7 +118,9 @@ typedef struct _PyThreadStateImpl { #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif - +#if _Py_TIER2 + _PyJitTracerState jit_tracer_state; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4abefd3b95d..4e1b15af42c 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -35,10 +35,18 @@ typedef struct _PyUOpInstruction{ #endif } _PyUOpInstruction; -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 1200 +// This is the length of the trace we translate initially. +#define UOP_MAX_TRACE_LENGTH 3000 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ff1d75c0cb1..7a33a5b84fd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -81,101 +81,107 @@ extern "C" { #define _CHECK_STACK_SPACE 357 #define _CHECK_STACK_SPACE_OPERAND 358 #define _CHECK_VALIDITY 359 -#define _COLD_EXIT 360 -#define _COMPARE_OP 361 -#define _COMPARE_OP_FLOAT 362 -#define _COMPARE_OP_INT 363 -#define _COMPARE_OP_STR 364 -#define _CONTAINS_OP 365 -#define _CONTAINS_OP_DICT 366 -#define _CONTAINS_OP_SET 367 +#define _COLD_DYNAMIC_EXIT 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 368 -#define _COPY_1 369 -#define _COPY_2 370 -#define _COPY_3 371 +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 372 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 373 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 374 -#define _DO_CALL_FUNCTION_EX 375 -#define _DO_CALL_KW 376 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 +#define _DYNAMIC_EXIT 378 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 377 +#define _ERROR_POP_N 379 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 378 -#define _EXPAND_METHOD_KW 379 -#define _FATAL_ERROR 380 +#define _EXPAND_METHOD 380 +#define _EXPAND_METHOD_KW 381 +#define _FATAL_ERROR 382 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 381 -#define _FOR_ITER_GEN_FRAME 382 -#define _FOR_ITER_TIER_TWO 383 +#define _FOR_ITER 383 +#define _FOR_ITER_GEN_FRAME 384 +#define _FOR_ITER_TIER_TWO 385 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 384 -#define _GUARD_CALLABLE_ISINSTANCE 385 -#define _GUARD_CALLABLE_LEN 386 -#define _GUARD_CALLABLE_LIST_APPEND 387 -#define _GUARD_CALLABLE_STR_1 388 -#define _GUARD_CALLABLE_TUPLE_1 389 -#define _GUARD_CALLABLE_TYPE_1 390 -#define _GUARD_DORV_NO_DICT 391 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 -#define _GUARD_GLOBALS_VERSION 393 -#define _GUARD_IS_FALSE_POP 394 -#define _GUARD_IS_NONE_POP 395 -#define _GUARD_IS_NOT_NONE_POP 396 -#define _GUARD_IS_TRUE_POP 397 -#define _GUARD_KEYS_VERSION 398 -#define _GUARD_NOS_DICT 399 -#define _GUARD_NOS_FLOAT 400 -#define _GUARD_NOS_INT 401 -#define _GUARD_NOS_LIST 402 -#define _GUARD_NOS_NOT_NULL 403 -#define _GUARD_NOS_NULL 404 -#define _GUARD_NOS_OVERFLOWED 405 -#define _GUARD_NOS_TUPLE 406 -#define _GUARD_NOS_UNICODE 407 -#define _GUARD_NOT_EXHAUSTED_LIST 408 -#define _GUARD_NOT_EXHAUSTED_RANGE 409 -#define _GUARD_NOT_EXHAUSTED_TUPLE 410 -#define _GUARD_THIRD_NULL 411 -#define _GUARD_TOS_ANY_SET 412 -#define _GUARD_TOS_DICT 413 -#define _GUARD_TOS_FLOAT 414 -#define _GUARD_TOS_INT 415 -#define _GUARD_TOS_LIST 416 -#define _GUARD_TOS_OVERFLOWED 417 -#define _GUARD_TOS_SLICE 418 -#define _GUARD_TOS_TUPLE 419 -#define _GUARD_TOS_UNICODE 420 -#define _GUARD_TYPE_VERSION 421 -#define _GUARD_TYPE_VERSION_AND_LOCK 422 -#define _HANDLE_PENDING_AND_DEOPT 423 +#define _GUARD_BINARY_OP_EXTEND 386 +#define _GUARD_CALLABLE_ISINSTANCE 387 +#define _GUARD_CALLABLE_LEN 388 +#define _GUARD_CALLABLE_LIST_APPEND 389 +#define _GUARD_CALLABLE_STR_1 390 +#define _GUARD_CALLABLE_TUPLE_1 391 +#define _GUARD_CALLABLE_TYPE_1 392 +#define _GUARD_DORV_NO_DICT 393 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 394 +#define _GUARD_GLOBALS_VERSION 395 +#define _GUARD_IP_RETURN_GENERATOR 396 +#define _GUARD_IP_RETURN_VALUE 397 +#define _GUARD_IP_YIELD_VALUE 398 +#define _GUARD_IP__PUSH_FRAME 399 +#define _GUARD_IS_FALSE_POP 400 +#define _GUARD_IS_NONE_POP 401 +#define _GUARD_IS_NOT_NONE_POP 402 +#define _GUARD_IS_TRUE_POP 403 +#define _GUARD_KEYS_VERSION 404 +#define _GUARD_NOS_DICT 405 +#define _GUARD_NOS_FLOAT 406 +#define _GUARD_NOS_INT 407 +#define _GUARD_NOS_LIST 408 +#define _GUARD_NOS_NOT_NULL 409 +#define _GUARD_NOS_NULL 410 +#define _GUARD_NOS_OVERFLOWED 411 +#define _GUARD_NOS_TUPLE 412 +#define _GUARD_NOS_UNICODE 413 +#define _GUARD_NOT_EXHAUSTED_LIST 414 +#define _GUARD_NOT_EXHAUSTED_RANGE 415 +#define _GUARD_NOT_EXHAUSTED_TUPLE 416 +#define _GUARD_THIRD_NULL 417 +#define _GUARD_TOS_ANY_SET 418 +#define _GUARD_TOS_DICT 419 +#define _GUARD_TOS_FLOAT 420 +#define _GUARD_TOS_INT 421 +#define _GUARD_TOS_LIST 422 +#define _GUARD_TOS_OVERFLOWED 423 +#define _GUARD_TOS_SLICE 424 +#define _GUARD_TOS_TUPLE 425 +#define _GUARD_TOS_UNICODE 426 +#define _GUARD_TYPE_VERSION 427 +#define _GUARD_TYPE_VERSION_AND_LOCK 428 +#define _HANDLE_PENDING_AND_DEOPT 429 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 424 -#define _INIT_CALL_PY_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS_0 426 -#define _INIT_CALL_PY_EXACT_ARGS_1 427 -#define _INIT_CALL_PY_EXACT_ARGS_2 428 -#define _INIT_CALL_PY_EXACT_ARGS_3 429 -#define _INIT_CALL_PY_EXACT_ARGS_4 430 -#define _INSERT_NULL 431 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 430 +#define _INIT_CALL_PY_EXACT_ARGS 431 +#define _INIT_CALL_PY_EXACT_ARGS_0 432 +#define _INIT_CALL_PY_EXACT_ARGS_1 433 +#define _INIT_CALL_PY_EXACT_ARGS_2 434 +#define _INIT_CALL_PY_EXACT_ARGS_3 435 +#define _INIT_CALL_PY_EXACT_ARGS_4 436 +#define _INSERT_NULL 437 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -185,177 +191,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 432 +#define _IS_NONE 438 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 433 -#define _ITER_CHECK_RANGE 434 -#define _ITER_CHECK_TUPLE 435 -#define _ITER_JUMP_LIST 436 -#define _ITER_JUMP_RANGE 437 -#define _ITER_JUMP_TUPLE 438 -#define _ITER_NEXT_LIST 439 -#define _ITER_NEXT_LIST_TIER_TWO 440 -#define _ITER_NEXT_RANGE 441 -#define _ITER_NEXT_TUPLE 442 -#define _JUMP_TO_TOP 443 +#define _ITER_CHECK_LIST 439 +#define _ITER_CHECK_RANGE 440 +#define _ITER_CHECK_TUPLE 441 +#define _ITER_JUMP_LIST 442 +#define _ITER_JUMP_RANGE 443 +#define _ITER_JUMP_TUPLE 444 +#define _ITER_NEXT_LIST 445 +#define _ITER_NEXT_LIST_TIER_TWO 446 +#define _ITER_NEXT_RANGE 447 +#define _ITER_NEXT_TUPLE 448 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 449 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 444 -#define _LOAD_ATTR_CLASS 445 +#define _LOAD_ATTR 450 +#define _LOAD_ATTR_CLASS 451 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 446 -#define _LOAD_ATTR_METHOD_LAZY_DICT 447 -#define _LOAD_ATTR_METHOD_NO_DICT 448 -#define _LOAD_ATTR_METHOD_WITH_VALUES 449 -#define _LOAD_ATTR_MODULE 450 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 451 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 452 -#define _LOAD_ATTR_PROPERTY_FRAME 453 -#define _LOAD_ATTR_SLOT 454 -#define _LOAD_ATTR_WITH_HINT 455 +#define _LOAD_ATTR_INSTANCE_VALUE 452 +#define _LOAD_ATTR_METHOD_LAZY_DICT 453 +#define _LOAD_ATTR_METHOD_NO_DICT 454 +#define _LOAD_ATTR_METHOD_WITH_VALUES 455 +#define _LOAD_ATTR_MODULE 456 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 457 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 458 +#define _LOAD_ATTR_PROPERTY_FRAME 459 +#define _LOAD_ATTR_SLOT 460 +#define _LOAD_ATTR_WITH_HINT 461 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 456 +#define _LOAD_BYTECODE 462 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 457 -#define _LOAD_CONST_INLINE_BORROW 458 -#define _LOAD_CONST_UNDER_INLINE 459 -#define _LOAD_CONST_UNDER_INLINE_BORROW 460 +#define _LOAD_CONST_INLINE 463 +#define _LOAD_CONST_INLINE_BORROW 464 +#define _LOAD_CONST_UNDER_INLINE 465 +#define _LOAD_CONST_UNDER_INLINE_BORROW 466 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 461 -#define _LOAD_FAST_0 462 -#define _LOAD_FAST_1 463 -#define _LOAD_FAST_2 464 -#define _LOAD_FAST_3 465 -#define _LOAD_FAST_4 466 -#define _LOAD_FAST_5 467 -#define _LOAD_FAST_6 468 -#define _LOAD_FAST_7 469 +#define _LOAD_FAST 467 +#define _LOAD_FAST_0 468 +#define _LOAD_FAST_1 469 +#define _LOAD_FAST_2 470 +#define _LOAD_FAST_3 471 +#define _LOAD_FAST_4 472 +#define _LOAD_FAST_5 473 +#define _LOAD_FAST_6 474 +#define _LOAD_FAST_7 475 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 470 -#define _LOAD_FAST_BORROW_0 471 -#define _LOAD_FAST_BORROW_1 472 -#define _LOAD_FAST_BORROW_2 473 -#define _LOAD_FAST_BORROW_3 474 -#define _LOAD_FAST_BORROW_4 475 -#define _LOAD_FAST_BORROW_5 476 -#define _LOAD_FAST_BORROW_6 477 -#define _LOAD_FAST_BORROW_7 478 +#define _LOAD_FAST_BORROW 476 +#define _LOAD_FAST_BORROW_0 477 +#define _LOAD_FAST_BORROW_1 478 +#define _LOAD_FAST_BORROW_2 479 +#define _LOAD_FAST_BORROW_3 480 +#define _LOAD_FAST_BORROW_4 481 +#define _LOAD_FAST_BORROW_5 482 +#define _LOAD_FAST_BORROW_6 483 +#define _LOAD_FAST_BORROW_7 484 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 479 -#define _LOAD_GLOBAL_BUILTINS 480 -#define _LOAD_GLOBAL_MODULE 481 +#define _LOAD_GLOBAL 485 +#define _LOAD_GLOBAL_BUILTINS 486 +#define _LOAD_GLOBAL_MODULE 487 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 482 -#define _LOAD_SMALL_INT_0 483 -#define _LOAD_SMALL_INT_1 484 -#define _LOAD_SMALL_INT_2 485 -#define _LOAD_SMALL_INT_3 486 -#define _LOAD_SPECIAL 487 +#define _LOAD_SMALL_INT 488 +#define _LOAD_SMALL_INT_0 489 +#define _LOAD_SMALL_INT_1 490 +#define _LOAD_SMALL_INT_2 491 +#define _LOAD_SMALL_INT_3 492 +#define _LOAD_SPECIAL 493 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 488 +#define _MAKE_CALLARGS_A_TUPLE 494 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 489 +#define _MAKE_WARM 495 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 490 -#define _MAYBE_EXPAND_METHOD_KW 491 -#define _MONITOR_CALL 492 -#define _MONITOR_CALL_KW 493 -#define _MONITOR_JUMP_BACKWARD 494 -#define _MONITOR_RESUME 495 +#define _MAYBE_EXPAND_METHOD 496 +#define _MAYBE_EXPAND_METHOD_KW 497 +#define _MONITOR_CALL 498 +#define _MONITOR_CALL_KW 499 +#define _MONITOR_JUMP_BACKWARD 500 +#define _MONITOR_RESUME 501 #define _NOP NOP -#define _POP_CALL 496 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_ONE 498 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 499 -#define _POP_CALL_TWO 500 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL 502 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_ONE 504 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL_TWO 506 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 507 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 502 -#define _POP_JUMP_IF_TRUE 503 +#define _POP_JUMP_IF_FALSE 508 +#define _POP_JUMP_IF_TRUE 509 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 504 -#define _POP_TOP_INT 505 -#define _POP_TOP_LOAD_CONST_INLINE 506 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 507 -#define _POP_TOP_NOP 508 -#define _POP_TOP_UNICODE 509 -#define _POP_TWO 510 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_FLOAT 510 +#define _POP_TOP_INT 511 +#define _POP_TOP_LOAD_CONST_INLINE 512 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 513 +#define _POP_TOP_NOP 514 +#define _POP_TOP_UNICODE 515 +#define _POP_TWO 516 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 517 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 512 +#define _PUSH_FRAME 518 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 513 -#define _PY_FRAME_GENERAL 514 -#define _PY_FRAME_KW 515 -#define _QUICKEN_RESUME 516 -#define _REPLACE_WITH_TRUE 517 +#define _PUSH_NULL_CONDITIONAL 519 +#define _PY_FRAME_GENERAL 520 +#define _PY_FRAME_KW 521 +#define _QUICKEN_RESUME 522 +#define _REPLACE_WITH_TRUE 523 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 518 -#define _SEND 519 -#define _SEND_GEN_FRAME 520 +#define _SAVE_RETURN_OFFSET 524 +#define _SEND 525 +#define _SEND_GEN_FRAME 526 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 521 -#define _STORE_ATTR 522 -#define _STORE_ATTR_INSTANCE_VALUE 523 -#define _STORE_ATTR_SLOT 524 -#define _STORE_ATTR_WITH_HINT 525 +#define _START_EXECUTOR 527 +#define _STORE_ATTR 528 +#define _STORE_ATTR_INSTANCE_VALUE 529 +#define _STORE_ATTR_SLOT 530 +#define _STORE_ATTR_WITH_HINT 531 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 526 -#define _STORE_FAST_0 527 -#define _STORE_FAST_1 528 -#define _STORE_FAST_2 529 -#define _STORE_FAST_3 530 -#define _STORE_FAST_4 531 -#define _STORE_FAST_5 532 -#define _STORE_FAST_6 533 -#define _STORE_FAST_7 534 +#define _STORE_FAST 532 +#define _STORE_FAST_0 533 +#define _STORE_FAST_1 534 +#define _STORE_FAST_2 535 +#define _STORE_FAST_3 536 +#define _STORE_FAST_4 537 +#define _STORE_FAST_5 538 +#define _STORE_FAST_6 539 +#define _STORE_FAST_7 540 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 535 -#define _STORE_SUBSCR 536 -#define _STORE_SUBSCR_DICT 537 -#define _STORE_SUBSCR_LIST_INT 538 -#define _SWAP 539 -#define _SWAP_2 540 -#define _SWAP_3 541 -#define _TIER2_RESUME_CHECK 542 -#define _TO_BOOL 543 +#define _STORE_SLICE 541 +#define _STORE_SUBSCR 542 +#define _STORE_SUBSCR_DICT 543 +#define _STORE_SUBSCR_LIST_INT 544 +#define _SWAP 545 +#define _SWAP_2 546 +#define _SWAP_3 547 +#define _TIER2_RESUME_CHECK 548 +#define _TO_BOOL 549 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 544 +#define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 545 +#define _TO_BOOL_STR 551 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 546 -#define _UNPACK_SEQUENCE_LIST 547 -#define _UNPACK_SEQUENCE_TUPLE 548 -#define _UNPACK_SEQUENCE_TWO_TUPLE 549 +#define _UNPACK_SEQUENCE 552 +#define _UNPACK_SEQUENCE_LIST 553 +#define _UNPACK_SEQUENCE_TUPLE 554 +#define _UNPACK_SEQUENCE_TWO_TUPLE 555 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 549 +#define MAX_UOP_ID 555 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 12487719969..d5a3c362d87 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,7 +11,7 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; @@ -19,7 +19,7 @@ extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -128,12 +128,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -256,7 +256,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, @@ -293,7 +293,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -315,6 +315,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -336,7 +337,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_COLD_EXIT] = 0, + [_COLD_DYNAMIC_EXIT] = 0, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -419,6 +425,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", @@ -443,6 +450,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", @@ -471,6 +479,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -1261,6 +1273,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: @@ -1305,6 +1319,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_IP__PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: + return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; default: return -1; } diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index fb4a441ca64..608ffdfad12 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3057,8 +3057,8 @@ def test_source_segment_missing_info(self): class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, transformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) + code, expected_code): + initial_ast = ast.parse(dedent(code)) expected_ast = ast.parse(dedent(expected_code)) transformer = transformer_class() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index e65556fb28f..f06c6cbda29 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -422,32 +422,6 @@ def testfunc(n, m): uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -847,38 +821,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_unicode_count), 1) self.assertIn("_COMPARE_OP_STR", uops) - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_sequential(self): def dummy12(x): return x - 1 @@ -907,6 +850,7 @@ def testfunc(n): largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_nested(self): def dummy12(x): return x + 3 @@ -937,6 +881,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls(self): def dummy12(x): return x + 3 @@ -972,6 +917,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls_different_order(self): # same as `several_calls` but with top-level calls reversed def dummy12(x): @@ -1008,6 +954,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_complex(self): def dummy0(x): return x @@ -1057,6 +1004,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_large_framesize(self): # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is # actually doing its job. Note that the resulting trace hits @@ -1118,6 +1066,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_recursion(self): def dummy15(x): while x > 0: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9d3248d972e..798f58737b1 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2253,9 +2253,10 @@ def frame_2_jit(expected: bool) -> None: def frame_3_jit() -> None: # JITs just before the last loop: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # 1 extra iteration for tracing. + for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst new file mode 100644 index 00000000000..40b9d19ee42 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -0,0 +1 @@ +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6514ca7f3cd..89e558b0fe8 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2661,7 +2661,8 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + // + 1 more due to one loop spent on tracing. + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 2)) < 0) { return 1; } diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fc3f5d9dde0..3aea2038fd1 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2432,6 +2432,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 + _PyJit_Tracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0cae3703d1d..b652973600c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,6 +17,7 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG +#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" @@ -260,7 +261,10 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), co, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), co); +#endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); _PyStackRef oldvalue = fast[i]; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 43198aaf8a7..b659ac80233 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -11,7 +11,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() - +#include "pycore_optimizer.h" // _PyJit_Tracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -1151,6 +1151,10 @@ func_dealloc(PyObject *self) if (_PyObject_ResurrectEnd(self)) { return; } +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), self, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), self); +#endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6ebd9ebdfce..2c798855a71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2938,8 +2938,8 @@ dummy_func( JUMP_BACKWARD_JIT, }; - tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { - #if ENABLE_SPECIALIZATION_FT + specializing tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); @@ -2953,25 +2953,21 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - ERROR_IF(optimized < 0); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - this_instr[1].counter = initial_jump_backoff_counter(); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -3017,6 +3013,10 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + goto stop_tracing; + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -3078,7 +3078,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + replaced inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -5245,19 +5245,40 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + } + #endif + tstate->jit_exit = exit; + TIER2_TO_TIER2(exit->executor); + } + + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); } - #endif - tstate->jit_exit = exit; - TIER2_TO_TIER2(exit->executor); + #endif + // Disabled for now (gh-139109) as it slows down dynamic code tremendously. + // Compile and jump to the cold dynamic executors in the future. + GOTO_TIER_ONE(frame->instr_ptr); } tier2 op(_CHECK_VALIDITY, (--)) { @@ -5369,7 +5390,8 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { @@ -5399,32 +5421,76 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); + // For control-flow guards, we don't want to increase the chain depth, as those don't actually + // represent deopts but rather just normal programs! + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. + // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. + // So setting it to anything else is wrong. + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(target); + } + } + + tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { + // TODO (gh-139109): This should be similar to _COLD_EXIT in the future. + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + } + + tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(_PUSH_FRAME); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(_PUSH_FRAME); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(YIELD_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(YIELD_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_GENERATOR); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_GENERATOR); + EXIT_IF(true); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); } label(pop_2_error) { @@ -5571,6 +5637,62 @@ dummy_func( DISPATCH(); } + label(record_previous_inst) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + if (full) { + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); + } + // Super instructions. Instruction deopted. There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + } + + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } + + label(stop_tracing) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 07d21575e3a..b76c9ec2811 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1004,6 +1004,8 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; +const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -1051,6 +1053,43 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#if _Py_TIER2 +// 0 for success, -1 for error. +static int +stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + int err = 0; + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { + err = _PyOptimizer_Optimize(frame, tstate); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Deal with backoffs + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit == NULL) { + // We hold a strong reference to the code object, so the instruction won't be freed. + if (err <= 0) { + _Py_BackoffCounter counter = _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter; + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + } + else { + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + } + } + else { + // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed + // to be valid to access. + if (err <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + _PyJit_FinalizeTracing(tstate); + return err; +} +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1180,9 +1219,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); # endif #else goto error; @@ -1191,9 +1230,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); # endif #else goto start_frame; @@ -1235,7 +1274,9 @@ _PyTier2Interpreter( tier2_start: next_uop = current_executor->trace; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1259,7 +1300,9 @@ _PyTier2Interpreter( uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index afdcbc563b2..05a2760671e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -93,11 +93,19 @@ # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) + # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) # define JUMP_TO_LABEL(name) \ do { \ Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ @@ -115,19 +123,36 @@ # endif # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #endif +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (0) +# define ENTER_TRACING() +# define LEAVE_TRACING() +#endif + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -164,11 +189,19 @@ do { \ DISPATCH_GOTO(); \ } +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ } #define DISPATCH_INLINED(NEW_FRAME) \ @@ -280,6 +313,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) @@ -366,12 +400,19 @@ do { \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ if (next_instr == NULL) { \ /* gh-140104: The exception handler expects frame->instr_ptr to after this_instr, not this_instr! */ \ next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ + if (keep_tracing_bit) { \ + assert(((_PyThreadStateImpl *)tstate)->jit_tracer_state.prev_state.code_curr_size == 2); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ DISPATCH(); \ } while (0) @@ -382,13 +423,23 @@ do { \ goto tier2_start; \ } while (0) -#define GOTO_TIER_ONE(TARGET) \ - do \ - { \ - tstate->current_executor = NULL; \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9ce0a9f8a4d..7ba2e9d0d92 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4189,6 +4189,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ + case _GET_LEN: { _PyStackRef obj; _PyStackRef len; @@ -7108,12 +7110,36 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + tstate->jit_exit = exit; + TIER2_TO_TIER2(exit->executor); + break; + } + + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), @@ -7121,8 +7147,8 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - tstate->jit_exit = exit; - TIER2_TO_TIER2(exit->executor); + + GOTO_TIER_ONE(frame->instr_ptr); break; } @@ -7419,7 +7445,8 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7460,37 +7487,101 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); } - exit->temperature = initial_temperature_backoff_counter(); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + GOTO_TIER_ONE(target); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); break; } + case _COLD_DYNAMIC_EXIT: { + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + break; + } + + case _GUARD_IP__PUSH_FRAME: { + #define OFFSET_OF__PUSH_FRAME ((0)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF__PUSH_FRAME; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF__PUSH_FRAME + break; + } + + case _GUARD_IP_YIELD_VALUE: { + #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_YIELD_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_YIELD_VALUE + break; + } + + case _GUARD_IP_RETURN_VALUE: { + #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_VALUE + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_GENERATOR + break; + } + + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 79328a7b725..a984da6dc91 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5476,6 +5476,10 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -7589,7 +7593,7 @@ /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { - #if ENABLE_SPECIALIZATION_FT + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); @@ -7645,30 +7649,20 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - JUMP_TO_LABEL(error); - } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -12265,5 +12259,75 @@ JUMP_TO_LABEL(error); DISPATCH(); } + LABEL(record_previous_inst) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b4b2bc5dc69..81e46a331e0 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,6 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" +#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ @@ -1785,6 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation diff --git a/Python/jit.c b/Python/jit.c index 279e1ce6a0d..7ab0f8ddd43 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -604,7 +604,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz unsigned char *code = memory; state.trampolines.mem = memory + code_size; unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT || trace[0].opcode == _COLD_DYNAMIC_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6dd443e1655..1b9196503b5 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -257,8 +257,270 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, +}; +#endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_table[256]; +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); @@ -266,6 +528,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); @@ -503,7 +767,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_table[256] = { +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -761,4 +1025,262 @@ static py_tail_call_funcptr instruction_funcptr_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, + [BUILD_LIST] = _TAIL_CALL_record_previous_inst, + [BUILD_MAP] = _TAIL_CALL_record_previous_inst, + [BUILD_SET] = _TAIL_CALL_record_previous_inst, + [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_STRING] = _TAIL_CALL_record_previous_inst, + [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, + [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, + [CACHE] = _TAIL_CALL_record_previous_inst, + [CALL] = _TAIL_CALL_record_previous_inst, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, + [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, + [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, + [CALL_KW] = _TAIL_CALL_record_previous_inst, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, + [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, + [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, + [CALL_LEN] = _TAIL_CALL_record_previous_inst, + [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_STR_1] = _TAIL_CALL_record_previous_inst, + [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, + [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, + [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, + [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, + [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, + [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, + [COPY] = _TAIL_CALL_record_previous_inst, + [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, + [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, + [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, + [DELETE_FAST] = _TAIL_CALL_record_previous_inst, + [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [DELETE_NAME] = _TAIL_CALL_record_previous_inst, + [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [DICT_MERGE] = _TAIL_CALL_record_previous_inst, + [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, + [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [END_FOR] = _TAIL_CALL_record_previous_inst, + [END_SEND] = _TAIL_CALL_record_previous_inst, + [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, + [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, + [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, + [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, + [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, + [FOR_ITER] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, + [GET_AITER] = _TAIL_CALL_record_previous_inst, + [GET_ANEXT] = _TAIL_CALL_record_previous_inst, + [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, + [GET_ITER] = _TAIL_CALL_record_previous_inst, + [GET_LEN] = _TAIL_CALL_record_previous_inst, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, + [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, + [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, + [IS_OP] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, + [LOAD_CONST] = _TAIL_CALL_record_previous_inst, + [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, + [LOAD_NAME] = _TAIL_CALL_record_previous_inst, + [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, + [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, + [MAKE_CELL] = _TAIL_CALL_record_previous_inst, + [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, + [MAP_ADD] = _TAIL_CALL_record_previous_inst, + [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, + [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, + [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, + [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [NOP] = _TAIL_CALL_record_previous_inst, + [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, + [POP_ITER] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [POP_TOP] = _TAIL_CALL_record_previous_inst, + [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, + [PUSH_NULL] = _TAIL_CALL_record_previous_inst, + [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, + [RERAISE] = _TAIL_CALL_record_previous_inst, + [RESERVED] = _TAIL_CALL_record_previous_inst, + [RESUME] = _TAIL_CALL_record_previous_inst, + [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, + [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, + [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [SEND] = _TAIL_CALL_record_previous_inst, + [SEND_GEN] = _TAIL_CALL_record_previous_inst, + [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, + [SET_ADD] = _TAIL_CALL_record_previous_inst, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, + [SET_UPDATE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [STORE_DEREF] = _TAIL_CALL_record_previous_inst, + [STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [STORE_NAME] = _TAIL_CALL_record_previous_inst, + [STORE_SLICE] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [SWAP] = _TAIL_CALL_record_previous_inst, + [TO_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, + [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, + [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, + [UNARY_NOT] = _TAIL_CALL_record_previous_inst, + [UNPACK_EX] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, + [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, + [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, +}; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 3b7e2dafab8..65007a256d0 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -29,11 +29,24 @@ #define MAX_EXECUTORS_SIZE 256 +// Trace too short, no progress: +// _START_EXECUTOR +// _MAKE_WARM +// _CHECK_VALIDITY +// _SET_IP +// is 4-5 instructions. +#define CODE_SIZE_NO_PROGRESS 5 +// We start with _START_EXECUTOR, _MAKE_WARM +#define CODE_SIZE_EMPTY 2 + #define _PyExecutorObject_CAST(op) ((_PyExecutorObject *)(op)) static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { + return false; + } if (instr->op.code == ENTER_EXECUTOR) { return true; } @@ -100,11 +113,11 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth); static int -uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, int curr_stackentries, +uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, + _PyExecutorObject **exec_ptr, bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. @@ -113,10 +126,10 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, // gh-137573: inlining this function causes stack overflows Py_NO_INLINE int _PyOptimizer_Optimize( - _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyExecutorObject **executor_ptr, int chain_depth) + _PyInterpreterFrame *frame, PyThreadState *tstate) { - _PyStackRef *stack_pointer = frame->stackpointer; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + int chain_depth = _tstate->jit_tracer_state.initial_state.chain_depth; PyInterpreterState *interp = _PyInterpreterState_GET(); if (!interp->jit) { // gh-140936: It is possible that interp->jit will become false during @@ -126,7 +139,9 @@ _PyOptimizer_Optimize( return 0; } assert(!interp->compiling); + assert(_tstate->jit_tracer_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED + assert(_tstate->jit_tracer_state.initial_state.func != NULL); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -134,18 +149,24 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = _PyFrame_GetCode(frame); - assert(PyCode_Check(code)); + PyCodeObject *code = (PyCodeObject *)_tstate->jit_tracer_state.initial_state.code; + _Py_CODEUNIT *start = _tstate->jit_tracer_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } - int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); + // One of our dependencies while tracing was invalidated. Not worth compiling. + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + interp->compiling = false; + return 0; + } + _PyExecutorObject *executor; + int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { interp->compiling = false; return err; } - assert(*executor_ptr != NULL); + assert(executor != NULL); if (progress_needed) { int index = get_index_for_executor(code, start); if (index < 0) { @@ -155,17 +176,21 @@ _PyOptimizer_Optimize( * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); + Py_DECREF(executor); interp->compiling = false; return 0; } - insert_executor(code, start, index, *executor_ptr); + insert_executor(code, start, index, executor); } else { - (*executor_ptr)->vm_data.code = NULL; + executor->vm_data.code = NULL; } - (*executor_ptr)->vm_data.chain_depth = chain_depth; - assert((*executor_ptr)->vm_data.valid); + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit != NULL) { + exit->executor = executor; + } + executor->vm_data.chain_depth = chain_depth; + assert(executor->vm_data.valid); interp->compiling = false; return 1; #else @@ -474,6 +499,14 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; +static const uint16_t +guard_ip_uop[MAX_UOP_ID + 1] = { + [_PUSH_FRAME] = _GUARD_IP__PUSH_FRAME, + [_RETURN_GENERATOR] = _GUARD_IP_RETURN_GENERATOR, + [_RETURN_VALUE] = _GUARD_IP_RETURN_VALUE, + [_YIELD_VALUE] = _GUARD_IP_YIELD_VALUE, +}; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -530,64 +563,19 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ + goto full; \ } -// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) -#define TRACE_STACK_PUSH() \ - if (trace_stack_depth >= TRACE_STACK_SIZE) { \ - DPRINTF(2, "Trace stack overflow\n"); \ - OPT_STAT_INC(trace_stack_overflow); \ - return 0; \ - } \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - trace_stack[trace_stack_depth].func = func; \ - trace_stack[trace_stack_depth].code = code; \ - trace_stack[trace_stack_depth].instr = instr; \ - trace_stack_depth++; -#define TRACE_STACK_POP() \ - if (trace_stack_depth <= 0) { \ - Py_FatalError("Trace stack underflow\n"); \ - } \ - trace_stack_depth--; \ - func = trace_stack[trace_stack_depth].func; \ - code = trace_stack[trace_stack_depth].code; \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - instr = trace_stack[trace_stack_depth].instr; - -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( +int +_PyJit_translate_single_bytecode_to_trace( + PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *next_instr, + bool stop_tracing) { - bool first = true; - PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(func)); - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; - struct { - PyFunctionObject *func; - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - bool jump_seen = false; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -596,410 +584,468 @@ translate_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; + bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; + int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); + _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; + _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; - for (;;) { - target = INSTR_IP(instr, code); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; - uint32_t opcode = instr->op.code; - uint32_t oparg = instr->op.arg; + target = Py_IsNone((PyObject *)old_code) + ? (int)(target_instr - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR) + : INSTR_IP(target_instr, old_code); - if (!first && instr == initial_instr) { - // We have looped around to the start: - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + // Rewind EXTENDED_ARG so that we see the whole thing. + // We must point to the first EXTENDED_ARG when deopting. + int oparg = _tstate->jit_tracer_state.prev_state.instr_oparg; + int opcode = this_instr->op.code; + int rewind_oparg = oparg; + while (rewind_oparg > 255) { + rewind_oparg >>= 8; + target--; + } + + int old_stack_level = _tstate->jit_tracer_state.prev_state.instr_stacklevel; + + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size < CODE_SIZE_NO_PROGRESS) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + if (has_dynamic_jump_taken && !needs_guard_ip) { + DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } + + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + + if (stop_tracing) { + ADD_TO_TRACE(_DEOPT, 0, 0, target); + goto done; + } + + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); + +#ifdef Py_DEBUG + if (oparg > 255) { + assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); + } +#endif + + // Skip over super instructions. + if (_tstate->jit_tracer_state.prev_state.instr_is_super) { + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + return 1; + } + + if (opcode == ENTER_EXECUTOR) { + goto full; + } + + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + goto done; + } + + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + if (frame != _tstate->jit_tracer_state.prev_state.instr_frame && !needs_guard_ip) { + DPRINTF(2, "Unsupported: unguardable jump taken\n"); + goto unsupported; + } + + if (oparg > 0xFFFF) { + DPRINTF(2, "Unsupported: oparg too large\n"); + goto unsupported; + } + + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { + unsupported: + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } goto done; } + } - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + if (opcode == NOP) { + return 1; + } - if (opcode == EXTENDED_ARG) { - instr++; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; - if (opcode == EXTENDED_ARG) { - instr--; + if (opcode == JUMP_FORWARD) { + return 1; + } + + if (opcode == EXTENDED_ARG) { + return 1; + } + + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; + + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + assert(!_PyErr_Occurred(tstate)); + + + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + max_length--; + } + + // _GUARD_IP leads to an exit. + max_length -= needs_guard_ip; + + RESERVE_RAW(expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); + + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + // Can be NULL for the entry frame. + if (old_code != NULL) { + _Py_BloomFilter_Add(dependencies, old_code); + } + + switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + { + _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); + _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; + assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); + int jump_happened = computed_jump_instr == next_instr; + assert(jump_happened == (target_instr[1].cache & 1)); + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + break; + } + case JUMP_BACKWARD_JIT: + // This is possible as the JIT might have re-activated after it was disabled + case JUMP_BACKWARD_NO_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + { + if ((next_instr != _tstate->jit_tracer_state.initial_state.close_loop_instr) && + (next_instr != _tstate->jit_tracer_state.initial_state.start_instr) && + _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS && + // For side exits, we don't want to terminate them early. + _tstate->jit_tracer_state.initial_state.exit == NULL && + // These are coroutines, and we want to unroll those usually. + opcode != JUMP_BACKWARD_NO_INTERRUPT) { + // We encountered a JUMP_BACKWARD but not to the top of our own loop. + // We don't want to continue tracing as we might get stuck in the + // inner loop. Instead, end the trace where the executor of the + // inner loop might start and let the traces rejoin. + OPT_STAT_INC(inner_loop); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, + _tstate->jit_tracer_state.initial_state.close_loop_instr, _tstate->jit_tracer_state.initial_state.start_instr); goto done; } - } - if (opcode == ENTER_EXECUTOR) { - // We have a couple of options here. We *could* peek "underneath" - // this executor and continue tracing, which could give us a longer, - // more optimizeable trace (at the expense of lots of duplicated - // tier two code). Instead, we choose to just end here and stitch to - // the other trace, which allows a side-exit traces to rejoin the - // "main" trace periodically (and also helps protect us against - // pathological behavior where the amount of tier two code explodes - // for a medium-length, branchy code path). This seems to work - // better in practice, but in the future we could be smarter about - // what we do here: - goto done; - } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target); + break; } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; + case RESUME: + case RESUME_CHECK: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + if (nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + assert(nuops > 0); + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + uint32_t orig_target = target; + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + target = orig_target; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - switch (opcode) { - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - { - RESERVE(1); - int counter = instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; - /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - For values in between, adjust proportionally. */ - if (jump_likely) { - confidence = confidence * bitcount / 16; + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } + else { + int extended_arg = orig_oparg > 255; + uint32_t jump_target = next_inst + orig_oparg + extended_arg; + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + if (is_for_iter_test[uop]) { + target = jump_target + 1; + } + } + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); } - else { - confidence = confidence * (16 - bitcount) / 16; - } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - target, _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); - OPT_STAT_INC(low_confidence); - goto done; - } - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - if (jump_likely) { - DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", - instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); - instr = target_instr; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); - goto top; - } - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); - break; - } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - case JUMP_BACKWARD: - case JUMP_BACKWARD_JIT: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - { - instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; - if (jump_seen) { - OPT_STAT_INC(inner_loop); - DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); - goto done; - } - jump_seen = true; - goto top; - } - - case JUMP_FORWARD: - { - RESERVE(0); - // This will emit two _SET_IP instructions; leave it to the optimizer - instr += oparg; - break; - } - - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; - - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { - // Check for trace stack underflow now: - // We can't bail e.g. in the middle of - // LOAD_CONST + _RETURN_VALUE. - if (trace_stack_depth == 0) { - DPRINTF(2, "Trace stack underflow\n"); - OPT_STAT_INC(trace_stack_underflow); - return 0; + operand = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + // Don't add nested code objects to the dependency. + // It causes endless re-traces. + if (new_func != NULL && !Py_IsNone((PyObject*)new_func) && !(new_code->co_flags & CO_NESTED)) { + operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); + } + else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { + operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); } } - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&instr[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&instr[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&instr[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } -#ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } -#endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); - } - - if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - TRACE_STACK_POP(); - /* Set the operand to the function or code object returned to, - * to assist optimization passes. (See _PUSH_FRAME below.) - */ - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (code != NULL) { - operand = (uintptr_t)code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - DPRINTF(2, - "Returning to %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - - if (uop == _PUSH_FRAME) { - assert(i + 1 == nuops); - if (opcode == FOR_ITER_GEN || - opcode == LOAD_ATTR_PROPERTY || - opcode == BINARY_OP_SUBSCR_GETITEM || - opcode == SEND_GEN) - { - DPRINTF(2, "Bailing due to dynamic target\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); - int func_version_offset = - offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) - // Add one to account for the actual opcode/oparg pair: - + 1; - uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyCodeObject *new_code = NULL; - PyFunctionObject *new_func = - _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); - DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", - (int)func_version, new_func, new_code); - if (new_code != NULL) { - if (new_code == code) { - // Recursive call, bail (we could be here forever). - DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", - PyUnicode_AsUTF8(new_code->co_qualname), - PyUnicode_AsUTF8(new_code->co_filename), - new_code->co_firstlineno); - OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - if (new_code->co_version != func_version) { - // func.__code__ was updated. - // Perhaps it may happen again, so don't bother tracing. - // TODO: Reason about this -- is it better to bail or not? - DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - // Increment IP to the return address - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; - TRACE_STACK_PUSH(); - _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function or code object, - * to assist optimization passes. - * We prefer setting it to the function - * but if that's not available but the code is available, - * use the code, setting the low bit so the optimizer knows. - */ - if (new_func != NULL) { - operand = (uintptr_t)new_func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - code = new_code; - func = new_func; - instr = _PyCode_CODE(code); - DPRINTF(2, - "Continuing in %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - DPRINTF(2, "Bail, new_code == NULL\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - instr++; - } - - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); - } + ADD_TO_TRACE(uop, oparg, operand, target); + trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next->op.code == STORE_FAST); + operand = next->op.arg; + } + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); + } + break; + } // End default - } // End switch (opcode) + } // End switch (opcode) - instr++; - // Add cache size for opcode - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - - if (opcode == CALL_LIST_APPEND) { - assert(instr->op.code == POP_TOP); - instr++; + if (needs_guard_ip) { + uint16_t guard_ip = guard_ip_uop[trace[trace_length-1].opcode]; + if (guard_ip == 0) { + DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); + Py_UNREACHABLE(); } - top: - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) - -done: - while (trace_stack_depth > 0) { - TRACE_STACK_POP(); + ADD_TO_TRACE(guard_ip, 0, (uintptr_t)next_instr, 0); } - assert(code == initial_code); - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + // Loop back to the start + int is_first_instr = _tstate->jit_tracer_state.initial_state.close_loop_instr == next_instr || + _tstate->jit_tracer_state.initial_state.start_instr == next_instr; + if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { + if (needs_guard_ip) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); + } + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } + DPRINTF(2, "Trace continuing\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 1; +done: + DPRINTF(2, "Trace done\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +full: + DPRINTF(2, "Trace full\n"); + if (!is_terminator(&_tstate->jit_tracer_state.code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + max_length = _tstate->jit_tracer_state.prev_state.code_max_size; + // We previously reversed one. + max_length += 1; + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + } + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +} + +// Returns 0 for do not enter tracing, 1 on enter tracing. +int +_PyJit_TryInitializeTracing( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, + _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, + _PyExitData *exit, int oparg) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // A recursive trace. + // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. + if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_EMPTY) { return 0; } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (oparg > 0xFFFF) { + return 0; } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + if (_tstate->jit_tracer_state.code_buffer == NULL) { + _tstate->jit_tracer_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (_tstate->jit_tracer_state.code_buffer == NULL) { + // Don't error, just go to next instruction. + return 0; + } + } + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func == NULL) { + return 0; + } + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + DPRINTF(2, + "Tracing %s (%s:%d) at byte offset %d at chain depth %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(close_loop_instr, code), + chain_depth); +#endif + + add_to_trace(_tstate->jit_tracer_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); + add_to_trace(_tstate->jit_tracer_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; + _tstate->jit_tracer_state.initial_state.start_instr = start_instr; + _tstate->jit_tracer_state.initial_state.close_loop_instr = close_loop_instr; + _tstate->jit_tracer_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + _tstate->jit_tracer_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); + _tstate->jit_tracer_state.initial_state.exit = exit; + _tstate->jit_tracer_state.initial_state.stack_depth = curr_stackdepth; + _tstate->jit_tracer_state.initial_state.chain_depth = chain_depth; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = true; + _tstate->jit_tracer_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _tstate->jit_tracer_state.prev_state.instr = curr_instr; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = curr_stackdepth; + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); + _tstate->jit_tracer_state.initial_state.jump_backward_instr = curr_instr; + + if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) { + close_loop_instr[1].counter = trigger_backoff_counter(); + } + _Py_BloomFilter_Init(&_tstate->jit_tracer_state.prev_state.dependencies); + return 1; } +void +_PyJit_FinalizeTracing(PyThreadState *tstate) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + Py_CLEAR(_tstate->jit_tracer_state.initial_state.code); + Py_CLEAR(_tstate->jit_tracer_state.initial_state.func); + Py_CLEAR(_tstate->jit_tracer_state.prev_state.instr_code); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; +} + + #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP @@ -1018,20 +1064,21 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } return exit_count; } -static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +static void make_exit(_PyUOpInstruction *inst, int opcode, int target, bool is_control_flow) { inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; + inst->operand1 = is_control_flow; #ifdef Py_STATS inst->execution_count = 0; #endif @@ -1075,15 +1122,17 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _HANDLE_PENDING_AND_DEOPT; } int32_t jump_target = target; - if (is_for_iter_test[opcode]) { - /* Target the POP_TOP immediately after the END_FOR, - * leaving only the iterator on the stack. */ - int extended_arg = inst->oparg > 255; - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; - jump_target = next_inst + inst->oparg + 1; + if ( + opcode == _GUARD_IP__PUSH_FRAME || + opcode == _GUARD_IP_RETURN_VALUE || + opcode == _GUARD_IP_YIELD_VALUE || + opcode == _GUARD_IP_RETURN_GENERATOR + ) { + exit_op = _DYNAMIC_EXIT; } + bool is_control_flow = (opcode == _GUARD_IS_FALSE_POP || opcode == _GUARD_IS_TRUE_POP || is_for_iter_test[opcode]); if (jump_target != current_jump_target || current_exit_op != exit_op) { - make_exit(&buffer[next_spare], exit_op, jump_target); + make_exit(&buffer[next_spare], exit_op, jump_target, is_control_flow); current_exit_op = exit_op; current_jump_target = jump_target; current_jump = next_spare; @@ -1099,7 +1148,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N, 0, false); buffer[next_spare].operand0 = target; next_spare++; } @@ -1157,7 +1206,9 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || + executor->trace[0].opcode == _COLD_EXIT || + executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1189,7 +1240,8 @@ sanity_check(_PyExecutorObject *executor) opcode == _DEOPT || opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + opcode == _ERROR_POP_N || + opcode == _DYNAMIC_EXIT); } } @@ -1202,7 +1254,7 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth) { int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); @@ -1212,10 +1264,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); + cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); - executor->exits[i].executor = cold; } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; @@ -1225,11 +1278,13 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; + exit->executor = opcode == _EXIT_TRACE ? cold : cold_dynamic; + exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); + exit->is_control_flow = (char)buffer[i].operand1; next_exit--; } } @@ -1291,38 +1346,32 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) static int uop_optimize( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyExecutorObject **exec_ptr, - int curr_stackentries, bool progress_needed) { - _PyBloomFilter dependencies; - _Py_BloomFilter_Init(&dependencies); - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->jit_uop_buffer == NULL) { - interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (interp->jit_uop_buffer == NULL) { - return 0; - } - } - _PyUOpInstruction *buffer = interp->jit_uop_buffer; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + _PyUOpInstruction *buffer = _tstate->jit_tracer_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; + int curr_stackentries = _tstate->jit_tracer_state.initial_state.stack_depth; + int length = _tstate->jit_tracer_state.prev_state.code_curr_size; + if (length <= CODE_SIZE_NO_PROGRESS) { + return 0; } + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(frame, buffer, - length, - curr_stackentries, &dependencies); + length = _Py_uop_analyze_and_optimize( + _tstate->jit_tracer_state.initial_state.func, + buffer,length, + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1345,14 +1394,14 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops( + buffer, length, dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); if (executor == NULL) { return -1; } assert(length <= UOP_MAX_TRACE_LENGTH); // Check executor coldness - PyThreadState *tstate = PyThreadState_Get(); // It's okay if this ends up going negative. if (--tstate->interp->executor_creation_counter == 0) { _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); @@ -1539,6 +1588,35 @@ _PyExecutor_GetColdExecutor(void) return cold; } +_PyExecutorObject * +_PyExecutor_GetColdDynamicExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_dynamic_executor != NULL) { + assert(interp->cold_dynamic_executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); + return interp->cold_dynamic_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_DYNAMIC_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_dynamic_executor = cold; + return cold; +} + void _PyExecutor_ClearExit(_PyExitData *exit) { @@ -1546,7 +1624,12 @@ _PyExecutor_ClearExit(_PyExitData *exit) return; } _PyExecutorObject *old = exit->executor; - exit->executor = _PyExecutor_GetColdExecutor(); + if (exit->is_dynamic) { + exit->executor = _PyExecutor_GetColdDynamicExecutor(); + } + else { + exit->executor = _PyExecutor_GetColdExecutor(); + } Py_DECREF(old); } @@ -1648,6 +1731,18 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_Executors_InvalidateAll(interp, is_invalidation); } +void +_PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (bloom_filter_may_contain(&_tstate->jit_tracer_state.prev_state.dependencies, &obj_filter)) + { + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = false; + } +} /* Invalidate all executors */ void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) @@ -1777,7 +1872,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else - fprintf(out, " %s\n", i, opname); + fprintf(out, " %s op0=%" PRIu64 "\n", i, opname, inst->operand0); #endif if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { break; @@ -1787,6 +1882,8 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) fprintf(out, "]\n\n"); /* Write all the outgoing edges */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1797,10 +1894,10 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != NULL) { + if (exit != NULL && exit->executor != cold && exit->executor != cold_dynamic) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a6add301ccb..8d7b734e17c 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -142,8 +142,10 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +#define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) + #define WITHIN_STACK_BOUNDS() \ - (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + (CURRENT_FRAME_IS_INIT_SHIM() || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -267,7 +269,7 @@ static PyCodeObject * get_current_code_object(JitOptContext *ctx) { - return (PyCodeObject *)ctx->frame->func->func_code; + return (PyCodeObject *)ctx->frame->code; } static PyObject * @@ -298,10 +300,6 @@ optimize_uops( JitOptContext context; JitOptContext *ctx = &context; uint32_t opcode = UINT16_MAX; - int curr_space = 0; - int max_space = 0; - _PyUOpInstruction *first_valid_check_stack = NULL; - _PyUOpInstruction *corresponding_check_stack = NULL; // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -320,13 +318,18 @@ optimize_uops( ctx->frame = frame; _PyUOpInstruction *this_instr = NULL; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; + for (int i = 0; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptRef *stack_pointer = ctx->frame->stack_pointer; + + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + stack_pointer = ctx->frame->stack_pointer; + } #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -345,9 +348,11 @@ optimize_uops( Py_UNREACHABLE(); } assert(ctx->frame != NULL); - DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); - ctx->frame->stack_pointer = stack_pointer; - assert(STACK_LEVEL() >= 0); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -355,27 +360,21 @@ optimize_uops( } if (ctx->contradiction) { // Attempted to push a "bottom" (contradiction) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. + // This means that the abstract interpreter has optimized to trace + // to an unreachable estate. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off + // bottom usually indicates an optimizer bug, so we are probably better off // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); + OPT_STAT_INC(optimizer_contradiction); return 0; } /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - if (first_valid_check_stack != NULL) { - assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); - assert(max_space > 0); - assert(max_space <= INT_MAX); - assert(max_space <= INT32_MAX); - first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand0 = max_space; - } return trace_len; error: @@ -460,6 +459,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _NOP; } break; + case _EXIT_TRACE: default: { // Cancel out pushes and pops, repeatedly. So: @@ -493,7 +493,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = opcode == _PUSH_FRAME; + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -503,10 +503,14 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[last_set_ip].opcode = _SET_IP; last_set_ip = -1; } + if (opcode == _EXIT_TRACE) { + return pc + 1; + } break; } case _JUMP_TO_TOP: - case _EXIT_TRACE: + case _DYNAMIC_EXIT: + case _DEOPT: return pc + 1; } } @@ -518,7 +522,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - _PyInterpreterFrame *frame, + PyFunctionObject *func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -528,8 +532,8 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_attempts); length = optimize_uops( - _PyFrame_GetFunction(frame), buffer, - length, curr_stacklen, dependencies); + func, buffer, + length, curr_stacklen, dependencies); if (length == 0) { return length; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index da3d3c96bc1..06fa8a4522a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -342,7 +342,6 @@ dummy_func(void) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } } @@ -752,8 +751,14 @@ dummy_func(void) { } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -764,8 +769,20 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = PyJitRef_NULL; - ctx->done = true; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + /* Push self onto stack of shim */ + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -773,42 +790,65 @@ dummy_func(void) { JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - RELOAD_STACK(); res = temp; } op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; } - op(_YIELD_VALUE, (unused -- value)) { - value = sym_new_unknown(ctx); + op(_YIELD_VALUE, (retval -- value)) { + // Mimics PyStackRef_MakeHeapSafe in the interpreter. + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + DEAD(retval); + SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + RELOAD_STACK(); + value = temp; } op(_GET_ITER, (iterable -- iter, index_or_null)) { @@ -835,8 +875,6 @@ dummy_func(void) { } op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; } op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { @@ -848,38 +886,29 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - assert(PyFunction_Check(func)); - ctx->frame->func = func; - /* Stack space handling */ - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - // won't fit in signed 32-bit int - ctx->done = true; - break; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + // No need to re-add to dependencies here. Already + // handled by the tracer. + ctx->frame->func = func; } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + // Fixed calls don't need IP guards. + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; } op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { @@ -1024,6 +1053,10 @@ dummy_func(void) { ctx->done = true; } + op(_DEOPT, (--)) { + ctx->done = true; + } + op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b08099d8e2f..01263fe8c7a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -280,7 +280,6 @@ int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; @@ -1116,16 +1115,24 @@ JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; res = temp; stack_pointer[0] = res; stack_pointer += 1; @@ -1167,9 +1174,28 @@ } case _YIELD_VALUE: { + JitOptRef retval; JitOptRef value; - value = sym_new_unknown(ctx); - stack_pointer[-1] = value; + retval = stack_pointer[-1]; + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + value = temp; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2103,6 +2129,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 */ + case _GET_LEN: { JitOptRef obj; JitOptRef len; @@ -2557,8 +2585,6 @@ } case _CHECK_STACK_SPACE: { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; break; } @@ -2601,34 +2627,26 @@ new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - assert(PyFunction_Check(func)); - ctx->frame->func = func; - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - ctx->done = true; - break; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + ctx->frame->func = func; } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - else if (corresponding_check_stack) { - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; break; } @@ -2761,9 +2779,24 @@ } case _CREATE_INIT_FRAME: { + JitOptRef *args; + JitOptRef self; JitOptRef init_frame; - init_frame = PyJitRef_NULL; - ctx->done = true; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2948,8 +2981,13 @@ case _PY_FRAME_KW: { JitOptRef new_frame; - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3005,17 +3043,19 @@ case _RETURN_GENERATOR: { JitOptRef res; - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3265,6 +3305,10 @@ break; } + case _DYNAMIC_EXIT: { + break; + } + case _CHECK_VALIDITY: { break; } @@ -3399,6 +3443,7 @@ } case _DEOPT: { + ctx->done = true; break; } @@ -3418,3 +3463,23 @@ break; } + case _COLD_DYNAMIC_EXIT: { + break; + } + + case _GUARD_IP__PUSH_FRAME: { + break; + } + + case _GUARD_IP_YIELD_VALUE: { + break; + } + + case _GUARD_IP_RETURN_VALUE: { + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 01cff0b014c..8a71eff465e 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -817,9 +817,14 @@ _Py_uop_frame_new( JitOptRef *args, int arg_len) { - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { + ctx->done = true; + ctx->out_of_space = true; + OPT_STAT_INC(optimizer_frame_overflow); + return NULL; + } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - + frame->code = co; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; @@ -901,13 +906,42 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) } int -_Py_uop_frame_pop(JitOptContext *ctx) +_Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries) { _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + if (ctx->curr_frame_depth >= 1) { + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + // We returned to the correct code. Nothing to do here. + if (co == ctx->frame->code) { + return 0; + } + // Else: the code we recorded doesn't match the code we *think* we're + // returning to. We could trace anything, we can't just return to the + // old frame. We have to restore what the tracer recorded + // as the traced next frame. + // Remove the current frame, and later swap it out with the right one. + else { + ctx->curr_frame_depth--; + } + } + // Else: trace stack underflow. + + // This handles swapping out frames. + assert(curr_stackentries >= 1); + // -1 to stackentries as we push to the stack our return value after this. + _Py_UOpsAbstractFrame *new_frame = _Py_uop_frame_new(ctx, co, curr_stackentries - 1, NULL, 0); + if (new_frame == NULL) { + ctx->done = true; + return 1; + } + + ctx->curr_frame_depth++; + ctx->frame = new_frame; return 0; } diff --git a/Python/pystate.c b/Python/pystate.c index 341c680a403..c12a1418e74 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -552,10 +552,6 @@ init_interpreter(PyInterpreterState *interp, _Py_brc_init_state(interp); #endif -#ifdef _Py_TIER2 - // Ensure the buffer is to be set as NULL. - interp->jit_uop_buffer = NULL; -#endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); interp->asyncio_tasks_lock = (PyMutex){0}; @@ -805,10 +801,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_uop_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE); - interp->jit_uop_buffer = NULL; - } #endif _PyAST_Fini(interp); _PyAtExit_Fini(interp); @@ -831,6 +823,14 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(cold->vm_data.warm); _PyExecutor_Free(cold); } + + struct _PyExecutorObject *cold_dynamic = interp->cold_dynamic_executor; + if (cold_dynamic != NULL) { + interp->cold_dynamic_executor = NULL; + assert(cold_dynamic->vm_data.valid); + assert(cold_dynamic->vm_data.warm); + _PyExecutor_Free(cold_dynamic); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1501,6 +1501,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; +#ifdef _Py_TIER2 + _tstate->jit_tracer_state.code_buffer = NULL; +#endif tstate->delete_later = NULL; llist_init(&_tstate->mem_free_queue); @@ -1807,6 +1810,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) assert(tstate_impl->refcounts.values == NULL); #endif +#if _Py_TIER2 + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->jit_tracer_state.code_buffer != NULL) { + _PyObject_VirtualFree(_tstate->jit_tracer_state.code_buffer, UOP_BUFFER_SIZE); + _tstate->jit_tracer_state.code_buffer = NULL; + } +#endif + HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4621ad250f4..bd4a8cf0d3e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -359,6 +359,7 @@ Parser/parser.c - soft_keywords - Parser/lexer/lexer.c - type_comment_prefix - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - +Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - Python/codecs.c - Py_hexdigits - Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfba..d39013db4f7 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -34,6 +34,8 @@ class Properties: side_exit: bool pure: bool uses_opcode: bool + needs_guard_ip: bool + unpredictable_jump: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -75,6 +77,8 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties), + unpredictable_jump=any(p.unpredictable_jump for p in properties), ) @property @@ -102,6 +106,8 @@ def infallible(self) -> bool: side_exit=False, pure=True, no_save_ip=False, + needs_guard_ip=False, + unpredictable_jump=False, ) @@ -692,6 +698,11 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJit_TryInitializeTracing", + "_Py_unset_eval_breaker_bit", + "_Py_set_eval_breaker_bit", + "trigger_backoff_counter", ) @@ -882,6 +893,46 @@ def stmt_escapes(stmt: Stmt) -> bool: else: assert False, "Unexpected statement type" +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: + if not stmts: + return False, branches_seen + predict = False + seen = 0 + for st in stmts: + predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) + predict = predict or predict_body + seen += seen_body + return predict, seen + +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: + if isinstance(stmt, BlockStmt): + return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "JUMPBY": + return True, branches_seen + return False, branches_seen + elif isinstance(stmt, IfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + 1 + return predict, seen + 1 + elif isinstance(stmt, MacroIfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + return predict, seen + elif isinstance(stmt, ForStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + elif isinstance(stmt, WhileStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + else: + assert False, f"Unexpected statement type {stmt}" + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) @@ -909,6 +960,8 @@ def compute_properties(op: parser.CodeDef) -> Properties: escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) + unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) return Properties( escaping_calls=escaping_calls, escapes=escapes, @@ -932,6 +985,11 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=(isinstance(op, parser.InstDef) + and (unpredictable_jump and "replaced" not in op.annotations)) + or variable_used(op, "LOAD_IP") + or variable_used(op, "DISPATCH_INLINED"), + unpredictable_jump=unpredictable_jump, ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 61e855eb003..0b5f764ec52 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,6 +7,7 @@ analysis_error, Label, CodeSection, + Uop, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable @@ -107,8 +108,9 @@ class Emitter: labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] cannot_escape: bool + jump_prefix: str - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, jump_prefix: str = ""): self._replacers = { "EXIT_IF": self.exit_if, "AT_END_EXIT_IF": self.exit_if_after, @@ -131,6 +133,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = self.out = out self.labels = labels self.cannot_escape = cannot_escape + self.jump_prefix = jump_prefix def dispatch( self, @@ -167,7 +170,7 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) @@ -198,10 +201,10 @@ def exit_if_after( def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"JUMP_TO_LABEL(error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(error);" def error_if( self, @@ -421,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit("JUMP_TO_LABEL(") + self.out.emit(f"{self.jump_prefix}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") @@ -731,6 +734,10 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.unpredictable_jump: + flags.append("HAS_UNPREDICTABLE_JUMP_FLAG") + if p.needs_guard_ip: + flags.append("HAS_NEEDS_GUARD_IP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b381233..21ae785a0ec 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -56,6 +56,8 @@ "ERROR_NO_POP", "NO_SAVE_IP", "PERIODIC", + "UNPREDICTABLE_JUMP", + "NEEDS_GUARD_IP", ] @@ -201,7 +203,7 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("struct opcode_metadata {\n") out.emit("uint8_t valid_entry;\n") out.emit("uint8_t instr_format;\n") - out.emit("uint16_t flags;\n") + out.emit("uint32_t flags;\n") out.emit("};\n\n") out.emit( f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 324ef2773ab..36fa1d7fa49 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -31,6 +31,16 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for target in targets: out.emit(target) out.emit("};\n") + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&record_previous_inst,\n" + out.emit("#if _Py_TIER2\n") + out.emit("static void *opcode_tracing_targets_table[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: @@ -38,7 +48,9 @@ def function_proto(name: str) -> str: def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256];\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -60,7 +72,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256] = {\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -68,6 +80,16 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + + # Emit the tracing dispatch table. + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 1bb5f48658d..ac3e6b94afe 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["IP_OFFSET_OF"] = self.ip_offset_of def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -134,10 +135,30 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def ip_offset_of( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert uop.name.startswith("_GUARD_IP") + # LPAREN + next(tkn_iter) + tok = next(tkn_iter) + self.emit(f" OFFSET_OF_{tok.text};\n") + # RPAREN + next(tkn_iter) + # SEMI + next(tkn_iter) + return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]]) -> Stack: locals: dict[str, Local] = {} try: + if name_offset_pair := offset_strs.get(uop.name): + emitter.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") emitter.out.start_line() if uop.properties.oparg: emitter.emit("oparg = CURRENT_OPARG();\n") @@ -158,6 +179,8 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: idx += 1 _, storage = emitter.emit_tokens(uop, storage, None, False) storage.flush(emitter.out) + if name_offset_pair: + emitter.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None return storage.stack @@ -165,6 +188,29 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) +def populate_offset_strs(analysis: Analysis) -> dict[str, tuple[str, str]]: + offset_strs: dict[str, tuple[str, str]] = {} + for name, uop in analysis.uops.items(): + if not f"_GUARD_IP_{name}" in analysis.uops: + continue + tkn_iter = uop.body.tokens() + found = False + offset_str = "" + for token in tkn_iter: + if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": + if found: + raise analysis_error("Cannot have two LOAD_IP in a guarded single uop.", uop.body.open) + offset = [] + while token.kind != "SEMI": + offset.append(token.text) + token = next(tkn_iter) + # 1: to remove the LOAD_IP text + offset_str = "".join(offset[1:]) + found = True + assert offset_str + offset_strs[f"_GUARD_IP_{name}"] = (name, offset_str) + return offset_strs + def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -179,7 +225,9 @@ def generate_tier2( ) out = CWriter(outfile, 2, lines) emitter = Tier2Emitter(out, analysis.labels) + offset_strs = populate_offset_strs(analysis) out.emit("\n") + for name, uop in analysis.uops.items(): if uop.properties.tier == 1: continue @@ -194,13 +242,15 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - stack = write_uop(uop, emitter, stack) + stack = write_uop(uop, emitter, stack, offset_strs) out.start_line() if not uop.properties.always_exits: out.emit("break;\n") out.start_line() out.emit("}") out.emit("\n\n") + + out.emit("\n") outfile.write("#undef TIER_TWO\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 1cc23837a72..0e0396e5143 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -23,13 +23,13 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") - out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + out.emit("const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 2f146014a1c..857e926d119 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -55,13 +55,10 @@ do { \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) -#undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ -do { \ - tstate->current_executor = NULL; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ -} while (0) +#undef GOTO_TIER_ONE_SETUP +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + _PyFrame_SetStackPointer(frame, stack_pointer); #undef LOAD_IP #define LOAD_IP(UNUSED) \ From 209eaff68c3b241c01aece14182cb9ced51526fc Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Fri, 14 Nov 2025 04:47:17 +1030 Subject: [PATCH 179/417] gh-137969: Fix double evaluation of `ForwardRef`s which rely on globals (#140974) --- Lib/annotationlib.py | 39 +++++++++------- Lib/test/test_annotationlib.py | 45 +++++++++++++++++++ ...-11-04-15-40-35.gh-issue-137969.9VZQVt.rst | 3 ++ 3 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 2166dbff0ee..33907b1fc2a 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -150,33 +150,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # Type parameters exist in their own scope, which is logically - # between the locals and the globals. We simulate this by adding - # them to the globals. Similar reasoning applies to nonlocals stored in cells. - if type_params is not None or isinstance(self.__cell__, dict): - globals = dict(globals) + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` if type_params is not None: for param in type_params: - globals[param.__name__] = param + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. if isinstance(self.__cell__, dict): - for cell_name, cell_value in self.__cell__.items(): + for cell_name, cell in self.__cell__.items(): try: - globals[cell_name] = cell_value.cell_contents + cell_value = cell.cell_contents except ValueError: pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 9f3275d5071..8208d0e9c94 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -2149,6 +2149,51 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + def test_fwdref_final_class(self): with self.assertRaises(TypeError): class C(ForwardRef): diff --git a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst new file mode 100644 index 00000000000..dfa582bdbc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst @@ -0,0 +1,3 @@ +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. From a486d452c78a7dfcd42561f6c151bf1fef0a756e Mon Sep 17 00:00:00 2001 From: Osama Abdelkader <78818069+osamakader@users.noreply.github.com> Date: Thu, 13 Nov 2025 20:05:28 +0100 Subject: [PATCH 180/417] gh-140601: Add ResourceWarning to iterparse when not closed (GH-140603) When iterparse() opens a file by filename and is not explicitly closed, emit a ResourceWarning to alert developers of the resource leak. Signed-off-by: Osama Abdelkader Co-authored-by: Serhiy Storchaka --- Doc/library/xml.etree.elementtree.rst | 4 ++ Doc/whatsnew/3.15.rst | 6 +++ Lib/test/test_xml_etree.py | 47 +++++++++++++++++++ Lib/xml/etree/ElementTree.py | 12 +++-- ...-10-25-22-55-07.gh-issue-140601.In3MlS.rst | 4 ++ 5 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 881708a4dd7..cbbc87b4721 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -656,6 +656,10 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. + .. versionchanged:: next + A :exc:`ResourceWarning` is now emitted if the iterator opened a file + and is not explicitly closed. + .. function:: parse(source, parser=None) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 895616e3049..31594a2e70b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1244,3 +1244,9 @@ that may require changes to your code. * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. + +* Resource warning is now emitted for unclosed + :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. + Use its :meth:`!close` method or the :func:`contextlib.closing` context + manager to close it. + (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 25c084c8b9c..87811199706 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1436,17 +1436,39 @@ def test_nonexistent_file(self): def test_resource_warnings_not_exhausted(self): # Not exhausting the iterator still closes the underlying file (bpo-43292) + # Not closing before del should emit ResourceWarning it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): + it.close() del it gc_collect() + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + del it + gc_collect() + # Not 'unclosed file'. + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: action, elem = next(it) self.assertEqual((action, elem.tag), ('end', 'element')) del it, elem gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) def test_resource_warnings_failed_iteration(self): self.addCleanup(os_helper.unlink, TESTFN) @@ -1461,16 +1483,41 @@ def test_resource_warnings_failed_iteration(self): next(it) self.assertEqual(str(cm.exception), 'junk after document element: line 1, column 12') + it.close() del cm, it gc_collect() + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertWarns(ResourceWarning) as wm: + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(TESTFN), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + def test_resource_warnings_exhausted(self): it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): list(it) + it.close() del it gc_collect() + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + list(it) + del it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + def test_close_not_exhausted(self): iterparse = ET.iterparse diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index dafe5b1b8a0..d8c0b1b6216 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1261,16 +1261,20 @@ def iterator(source): gen = iterator(source) class IterParseIterator(collections.abc.Iterator): __next__ = gen.__next__ + def close(self): + nonlocal close_source if close_source: source.close() + close_source = False gen.close() - def __del__(self): - # TODO: Emit a ResourceWarning if it was not explicitly closed. - # (When the close() method will be supported in all maintained Python versions.) + def __del__(self, _warn=warnings.warn): if close_source: - source.close() + try: + _warn(f"unclosed iterparse iterator {source.name!r}", ResourceWarning, stacklevel=2) + finally: + source.close() it = IterParseIterator() it.root = None diff --git a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst new file mode 100644 index 00000000000..72666bb8224 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst @@ -0,0 +1,4 @@ +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. From 4885ecfbda4cc792691e5d488ef6cb09727eb417 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Fri, 14 Nov 2025 04:18:54 +0100 Subject: [PATCH 181/417] gh-140790: pdb: Initialize instance variables in Pdb.__init__ (#140791) Initialize lineno, stack, curindex, curframe, currentbp, and _user_requested_quit attributes in `Pdb.__init__``. --- Lib/pdb.py | 10 ++++++++-- .../2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index fdc74198582..b799a113503 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -398,6 +398,12 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._current_task = None + self.lineno = None + self.stack = [] + self.curindex = 0 + self.curframe = None + self._user_requested_quit = False + def set_trace(self, frame=None, *, commands=None): Pdb._last_pdb_instance = self if frame is None: @@ -474,7 +480,7 @@ def forget(self): self.lineno = None self.stack = [] self.curindex = 0 - if hasattr(self, 'curframe') and self.curframe: + if self.curframe: self.curframe.f_globals.pop('__pdb_convenience_variables', None) self.curframe = None self.tb_lineno.clear() @@ -1493,7 +1499,7 @@ def checkline(self, filename, lineno, module_globals=None): """ # this method should be callable before starting debugging, so default # to "no globals" if there is no current frame - frame = getattr(self, 'curframe', None) + frame = self.curframe if module_globals is None: module_globals = frame.f_globals if frame else None line = linecache.getline(filename, lineno, module_globals) diff --git a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst new file mode 100644 index 00000000000..03856f0b9b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst @@ -0,0 +1 @@ +Initialize all Pdb's instance variables in ``__init__``, remove some hasattr/getattr From a4dd66275b62453bec055d730a8ce7173e519b6d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 10:38:49 +0100 Subject: [PATCH 182/417] gh-140550: Use a bool for the Py_mod_gil value (GH-141519) This needs a single bit, but was stored as a void* in the module struct. This didn't matter due to packing, but now that there's another bool in the struct, we can save a bit of memory by making md_gil a bool. Variables that changed type are renamed, to detect conflicts. --- Include/internal/pycore_moduleobject.h | 2 +- Lib/test/test_sys.py | 2 +- Objects/moduleobject.c | 15 ++++++++------- Python/import.c | 26 ++++++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 6eef6eaa5df..9a62daf6621 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -30,7 +30,7 @@ typedef struct { PyObject *md_name; bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif Py_ssize_t md_state_size; traverseproc md_state_traverse; diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 798f58737b1..2f169c1165d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1725,7 +1725,7 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - md_gil = 'P' + md_gil = '?' else: md_gil = '' check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 9dee03bdb5e..6c1c5f5eb89 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -178,7 +178,7 @@ new_module_notrack(PyTypeObject *mt) m->md_name = NULL; m->md_token_is_def = false; #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif m->md_state_size = 0; m->md_state_traverse = NULL; @@ -361,7 +361,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) m->md_token_is_def = true; module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif return (PyObject*)m; } @@ -380,7 +380,7 @@ module_from_def_and_spec( int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; int has_gil_slot = 0; - void *gil_slot = Py_MOD_GIL_USED; + bool requires_gil = true; int has_execution_slots = 0; const char *name; int ret; @@ -474,7 +474,7 @@ module_from_def_and_spec( name); goto error; } - gil_slot = cur_slot->value; + requires_gil = (cur_slot->value != Py_MOD_GIL_NOT_USED); has_gil_slot = 1; break; case Py_mod_abi: @@ -581,9 +581,9 @@ module_from_def_and_spec( mod->md_token = token; } #ifdef Py_GIL_DISABLED - mod->md_gil = gil_slot; + mod->md_requires_gil = requires_gil; #else - (void)gil_slot; + (void)requires_gil; #endif mod->md_exec = m_exec; } else { @@ -664,11 +664,12 @@ PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) int PyUnstable_Module_SetGIL(PyObject *module, void *gil) { + bool requires_gil = (gil != Py_MOD_GIL_NOT_USED); if (!PyModule_Check(module)) { PyErr_BadInternalCall(); return -1; } - ((PyModuleObject *)module)->md_gil = gil; + ((PyModuleObject *)module)->md_requires_gil = requires_gil; return 0; } #endif diff --git a/Python/import.c b/Python/import.c index 2afa7c15e6a..b05b40448d0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1017,9 +1017,10 @@ struct extensions_cache_value { _Py_ext_module_origin origin; #ifdef Py_GIL_DISABLED - /* The module's md_gil slot, for legacy modules that are reinitialized from - m_dict rather than calling their initialization function again. */ - void *md_gil; + /* The module's md_requires_gil member, for legacy modules that are + * reinitialized from m_dict rather than calling their initialization + * function again. */ + bool md_requires_gil; #endif }; @@ -1350,7 +1351,7 @@ static struct extensions_cache_value * _extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def, PyModInitFunction m_init, Py_ssize_t m_index, PyObject *m_dict, - _Py_ext_module_origin origin, void *md_gil) + _Py_ext_module_origin origin, bool requires_gil) { struct extensions_cache_value *value = NULL; void *key = NULL; @@ -1405,11 +1406,11 @@ _extensions_cache_set(PyObject *path, PyObject *name, /* m_dict is set by set_cached_m_dict(). */ .origin=origin, #ifdef Py_GIL_DISABLED - .md_gil=md_gil, + .md_requires_gil=requires_gil, #endif }; #ifndef Py_GIL_DISABLED - (void)md_gil; + (void)requires_gil; #endif if (init_cached_m_dict(newvalue, m_dict) < 0) { goto finally; @@ -1547,7 +1548,8 @@ _PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) } if (!PyModule_Check(module) || - ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + ((PyModuleObject *)module)->md_requires_gil) + { if (_PyEval_EnableGILPermanent(tstate)) { int warn_result = PyErr_WarnFormat( PyExc_RuntimeWarning, @@ -1725,7 +1727,7 @@ struct singlephase_global_update { Py_ssize_t m_index; PyObject *m_dict; _Py_ext_module_origin origin; - void *md_gil; + bool md_requires_gil; }; static struct extensions_cache_value * @@ -1784,7 +1786,7 @@ update_global_state_for_extension(PyThreadState *tstate, #endif cached = _extensions_cache_set( path, name, def, m_init, singlephase->m_index, m_dict, - singlephase->origin, singlephase->md_gil); + singlephase->origin, singlephase->md_requires_gil); if (cached == NULL) { // XXX Ignore this error? Doing so would effectively // mark the module as not loadable. @@ -1873,7 +1875,7 @@ reload_singlephase_extension(PyThreadState *tstate, if (def->m_base.m_copy != NULL) { // For non-core modules, fetch the GIL slot that was stored by // import_run_extension(). - ((PyModuleObject *)mod)->md_gil = cached->md_gil; + ((PyModuleObject *)mod)->md_requires_gil = cached->md_requires_gil; } #endif /* We can't set mod->md_def if it's missing, @@ -2128,7 +2130,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, .m_index=def->m_base.m_index, .origin=info->origin, #ifdef Py_GIL_DISABLED - .md_gil=((PyModuleObject *)mod)->md_gil, + .md_requires_gil=((PyModuleObject *)mod)->md_requires_gil, #endif }; // gh-88216: Extensions and def->m_base.m_copy can be updated @@ -2323,7 +2325,7 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, .origin=_Py_ext_module_origin_CORE, #ifdef Py_GIL_DISABLED /* Unused when m_dict == NULL. */ - .md_gil=NULL, + .md_requires_gil=false, #endif }; cached = update_global_state_for_extension( From 1e4e59bb3714ba7c6b6297f1a74e231b056f004c Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Fri, 14 Nov 2025 01:43:25 -0800 Subject: [PATCH 183/417] gh-116146: Add C-API to create module from spec and initfunc (GH-139196) Co-authored-by: Kumar Aditya Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/c-api/import.rst | 21 ++++ Doc/whatsnew/3.15.rst | 4 + Include/cpython/import.h | 7 ++ Lib/test/test_embed.py | 25 ++++ ...-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 + Programs/_testembed.c | 111 ++++++++++++++++++ Python/import.c | 74 ++++++++---- 7 files changed, 223 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b1..24e673d3d13 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -333,3 +333,24 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`!PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: next diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 31594a2e70b..9393b65ed8e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1080,6 +1080,10 @@ New features thread state. (Contributed by Victor Stinner in :gh:`139653`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + Changed C APIs -------------- diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cc..149a20af8b9 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1933f691a78..1078796eae8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -239,6 +239,31 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_create_module_from_initfunc(self): + out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") + if support.Py_GIL_DISABLED: + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + " has been enabled to load module 'embedded_ext'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + self.assertEqual(filtered_err_lines, []) + else: + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "my_test_extension.executed='yes'\n" + "my_test_extension.exec_slot_ran='yes'\n" + "\n" + "embedded_ext.executed='yes'\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst new file mode 100644 index 00000000000..be8043e26dd --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst @@ -0,0 +1,2 @@ +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d3600fecbe2..27224e508bd 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,6 +166,8 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { + // keep this as a single-phase initialization module; + // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } @@ -1894,8 +1896,16 @@ static int test_initconfig_exit(void) } +int +extension_module_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes"); +} + + static PyModuleDef_Slot extension_slots[] = { {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, extension_module_exec}, {0, NULL} }; @@ -2213,6 +2223,106 @@ static int test_repeated_init_and_inittab(void) return 0; } +static PyObject* +create_module(PyObject* self, PyObject* spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (!name) { + return NULL; + } + if (PyUnicode_EqualToUTF8(name, "my_test_extension")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + } + if (PyUnicode_EqualToUTF8(name, "embedded_ext")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + } + PyErr_Format(PyExc_LookupError, "static module %R not found", name); + Py_DECREF(name); + return NULL; +} + +static PyObject* +exec_module(PyObject* self, PyObject* mod) +{ + if (PyModule_Exec(mod) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef create_static_module_methods[] = { + {"create_module", create_module, METH_O, NULL}, + {"exec_module", exec_module, METH_O, NULL}, + {} +}; + +static struct PyModuleDef create_static_module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "create_static_module", + .m_size = 0, + .m_methods = create_static_module_methods, + .m_slots = extension_slots, +}; + +PyMODINIT_FUNC PyInit_create_static_module(void) { + return PyModuleDef_Init(&create_static_module_def); +} + +static int +test_create_module_from_initfunc(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + // Multi-phase initialization + L"import my_test_extension;" + L"print(my_test_extension);" + L"print(f'{my_test_extension.executed=}');" + L"print(f'{my_test_extension.exec_slot_ran=}');" + // Single-phase initialization + L"import embedded_ext;" + L"print(embedded_ext);" + L"print(f'{embedded_ext.executed=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("create_static_module", + &PyInit_create_static_module) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + int result = PyRun_SimpleString( + "import sys\n" + "from importlib.util import spec_from_loader\n" + "import create_static_module\n" + "class StaticExtensionImporter:\n" + " _ORIGIN = \"static-extension\"\n" + " @classmethod\n" + " def find_spec(cls, fullname, path, target=None):\n" + " if fullname in {'my_test_extension', 'embedded_ext'}:\n" + " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" + " return None\n" + " @staticmethod\n" + " def create_module(spec):\n" + " return create_static_module.create_module(spec)\n" + " @staticmethod\n" + " def exec_module(module):\n" + " create_static_module.exec_module(module)\n" + " module.executed = 'yes'\n" + "sys.meta_path.append(StaticExtensionImporter)\n" + ); + if (result < 0) { + fprintf(stderr, "PyRun_SimpleString() failed\n"); + return 1; + } + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2396,6 +2506,7 @@ static struct TestCase TestCases[] = { #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, {"test_gilstate_after_finalization", test_gilstate_after_finalization}, + {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {NULL, NULL} }; diff --git a/Python/import.c b/Python/import.c index b05b40448d0..9ab2d3b3552 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2364,8 +2364,23 @@ is_builtin(PyObject *name) return 0; } +static PyModInitFunction +lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) +{ + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { + return (PyModInitFunction)p->initfunc; + } + } + // not found + return NULL; +} + static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +create_builtin( + PyThreadState *tstate, PyObject *name, + PyObject *spec, + PyModInitFunction initfunc) { struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { @@ -2396,25 +2411,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) _extensions_cache_delete(info.path, info.name); } - struct _inittab *found = NULL; - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { - found = p; - break; - } - } - if (found == NULL) { - // not found - mod = Py_NewRef(Py_None); - goto finally; - } - - PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + PyModInitFunction p0 = initfunc; if (p0 == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, info.name, info.path)); - mod = import_add_module(tstate, info.name); - goto finally; + p0 = lookup_inittab_initfunc(&info); + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } } #ifdef Py_GIL_DISABLED @@ -2440,6 +2445,33 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return mod; } +PyObject* +PyImport_CreateModuleFromInitfunc( + PyObject *spec, PyObject *(*initfunc)(void)) +{ + if (initfunc == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "spec name must be string, not %T", name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin(tstate, name, spec, initfunc); + Py_DECREF(name); + return mod; +} /*****************************/ /* the builtin modules table */ @@ -3209,7 +3241,7 @@ bootstrap_imp(PyThreadState *tstate) } // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_CLEAR(name); Py_DECREF(spec); if (mod == NULL) { @@ -4369,7 +4401,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_DECREF(name); return mod; } From 181a2f4f2e3bed8dc6be5630e9bfb3362194ab3a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:59:19 +0200 Subject: [PATCH 184/417] gh-139596: Cease caching config.cache & ccache in GH Actions (#141451) --- .github/workflows/build.yml | 5 ----- .github/workflows/reusable-context.yml | 9 --------- .github/workflows/reusable-macos.yml | 3 --- .github/workflows/reusable-san.yml | 3 --- .github/workflows/reusable-ubuntu.yml | 3 --- .github/workflows/reusable-wasi.yml | 6 +----- .gitignore | 1 - 7 files changed, 1 insertion(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0f60c30ac8..8e15400e497 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -205,7 +205,6 @@ jobs: free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -237,7 +236,6 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -414,8 +412,6 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" @@ -600,7 +596,6 @@ jobs: uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d2668ddcac1..66c7cc47de0 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,9 +17,6 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool @@ -42,7 +39,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} run-docs: ${{ steps.changes.outputs.run-docs }} run-tests: ${{ steps.changes.outputs.run-tests }} @@ -100,8 +96,3 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d85c46b96f8..98d557ba1ea 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 7fe96d1b238..c601d0b7338 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -6,9 +6,6 @@ on: sanitizer: required: true type: string - config_hash: - required: true - type: string free-threading: description: Whether to use free-threaded mode required: false diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7b93b5f51b0..0c1ebe29ae3 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 8f412288f53..a309ef4e7f4 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,6 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string env: FORCE_COLOR: 1 @@ -53,7 +49,7 @@ jobs: - name: "Configure build Python" run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python + run: python3 Tools/wasm/wasi make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi configure-host -- --config-cache diff --git a/.gitignore b/.gitignore index 2bf4925647d..4ea2fd96554 100644 --- a/.gitignore +++ b/.gitignore @@ -135,7 +135,6 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ /jit_stencils*.h From 3bacae55980561cb99095a20a70c45d6174e056d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Nov 2025 11:13:24 +0100 Subject: [PATCH 185/417] gh-131510: Use PyUnstable_Unicode_GET_CACHED_HASH() (GH-141520) Replace code that directly accesses PyASCIIObject.hash with PyUnstable_Unicode_GET_CACHED_HASH(). Remove redundant "assert(PyUnicode_Check(op))" from PyUnstable_Unicode_GET_CACHED_HASH(), _PyASCIIObject_CAST() already implements the check. --- Include/cpython/unicodeobject.h | 1 - Include/internal/pycore_object.h | 3 +-- Objects/dictobject.c | 3 +-- Objects/typeobject.c | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 73e3bc44d6c..2853d24c34b 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -301,7 +301,6 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { /* Returns the cached hash, or -1 if not cached yet. */ static inline Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { - assert(PyUnicode_Check(op)); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); #else diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 980d6d7764b..fb50acd62da 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -863,8 +863,7 @@ static inline Py_hash_t _PyObject_HashFast(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 65eed151c28..14de21f3c67 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -400,8 +400,7 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); static inline Py_hash_t unicode_get_hash(PyObject *o) { - assert(PyUnicode_CheckExact(o)); - return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); + return PyUnstable_Unicode_GET_CACHED_HASH(o); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 58228d62485..61bcc21ce13 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6036,7 +6036,7 @@ static PyObject * update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) { _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); + assert(PyUnstable_Unicode_GET_CACHED_HASH(name) != -1); OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); // We're releasing this under the lock for simplicity sake because it's always a // exact unicode object or Py_None so it's safe to do so. From 5ac0b55ebc792936184f8e08697e60d5b3f8b946 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 11:22:18 +0100 Subject: [PATCH 186/417] gh-141376: Remove exceptions from `make smelly` (GH-141392) * Don't ignore initialized data and BSS * Remove exceptions for _init and _fini --- Tools/build/smelly.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 9a360412a73..424fa6ad4a1 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -21,8 +21,6 @@ }) IGNORED_EXTENSION = "_ctypes_test" -# Ignore constructor and destructor functions -IGNORED_SYMBOLS = {'_init', '_fini'} def is_local_symbol_type(symtype): @@ -34,19 +32,12 @@ def is_local_symbol_type(symtype): if symtype.islower() and symtype not in "uvw": return True - # Ignore the initialized data section (d and D) and the BSS data - # section. For example, ignore "__bss_start (type: B)" - # and "_edata (type: D)". - if symtype in "bBdD": - return True - return False def get_exported_symbols(library, dynamic=False): print(f"Check that {library} only exports symbols starting with Py or _Py") - # Only look at dynamic symbols args = ['nm', '--no-sort'] if dynamic: args.append('--dynamic') @@ -89,8 +80,6 @@ def get_smelly_symbols(stdout, dynamic=False): if is_local_symbol_type(symtype): local_symbols.append(result) - elif symbol in IGNORED_SYMBOLS: - local_symbols.append(result) else: smelly_symbols.append(result) From ef90261be508b97d682589aac8f00065a9585683 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:20:36 +0000 Subject: [PATCH 187/417] gh-141004: Document `PyOS_InterruptOccurred` (GH-141526) --- Doc/c-api/sys.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 336e3ef9640..ee73c1c8ada 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: From c10fa5be6167b1338ad194f9fe4be4782e025175 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 14 Nov 2025 09:22:36 -0500 Subject: [PATCH 188/417] gh-131229: Temporarily skip `test_basic_multiple_interpreters_deleted_no_reset` (GH-141552) This is a temporary band-aid to unblock other PRs. Co-authored-by: Kumar Aditya --- Lib/test/test_import/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index fe669bb04df..fd9750eae80 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3261,6 +3261,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * m_copy was copied from interp2 (was from interp1) # * module's global state was updated, not reset + @unittest.skip("gh-131229: This is suddenly very flaky") @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") @requires_subinterpreters def test_basic_multiple_interpreters_deleted_no_reset(self): From 8deaa9393eadf84e6e571be611e0c5a377abf7cd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 14 Nov 2025 16:49:28 +0200 Subject: [PATCH 189/417] gh-122255: Synchronize warnings in C and Python implementations of the warnings module (GH-122824) In the linecache module and in the Python implementation of the warnings module, a DeprecationWarning is issued when m.__loader__ differs from m.__spec__.loader (like in the C implementation of the warnings module). --- Lib/linecache.py | 63 +++++++++++++++---- Lib/test/test_linecache.py | 32 ++++++++-- Lib/test/test_warnings/__init__.py | 5 +- ...-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst | 4 ++ 4 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index ef3b2d9136b..b5bf9dbdd3c 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -224,21 +224,58 @@ def lazycache(filename, module_globals): def _make_lazycache_entry(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): return None - # Try for a __loader__, if available - if module_globals and '__name__' in module_globals: - spec = module_globals.get('__spec__') - name = getattr(spec, 'name', None) or module_globals['__name__'] - loader = getattr(spec, 'loader', None) - if loader is None: - loader = module_globals.get('__loader__') - get_source = getattr(loader, 'get_source', None) - if name and get_source: - def get_lines(name=name, *args, **kwargs): - return get_source(name, *args, **kwargs) - return (get_lines,) - return None + if module_globals is not None and not isinstance(module_globals, dict): + raise TypeError(f'module_globals must be a dict, not {type(module_globals).__qualname__}') + if not module_globals or '__name__' not in module_globals: + return None + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + if name is None: + return None + + loader = _bless_my_loader(module_globals) + if loader is None: + return None + + get_source = getattr(loader, 'get_source', None) + if get_source is None: + return None + + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) + return (get_lines,) + +def _bless_my_loader(module_globals): + # Similar to _bless_my_loader() in importlib._bootstrap_external, + # but always emits warnings instead of errors. + loader = module_globals.get('__loader__') + if loader is None and '__spec__' not in module_globals: + return None + spec = module_globals.get('__spec__') + + # The __main__ module has __spec__ = None. + if spec is None and module_globals.get('__name__') == '__main__': + return loader + + spec_loader = getattr(spec, 'loader', None) + if spec_loader is None: + import warnings + warnings.warn( + 'Module globals is missing a __spec__.loader', + DeprecationWarning) + return loader + + assert spec_loader is not None + if loader is not None and loader != spec_loader: + import warnings + warnings.warn( + 'Module globals; __loader__ != __spec__.loader', + DeprecationWarning) + return loader + + return spec_loader def _register_code(code, string, name): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 02f65338428..fcd94edc611 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -259,22 +259,44 @@ def raise_memoryerror(*args, **kwargs): def test_loader(self): filename = 'scheme://path' - for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': None} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + for loader in object(), NoSourceLoader(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': loader} - self.assertEqual(linecache.getlines(filename, module_globals), []) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), []) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} - self.assertEqual(linecache.getlines(filename, module_globals), - ['source for a.b.c\n']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') - for spec in (None, object(), ModuleSpec('', FakeLoader())): + for spec in None, object(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), '__spec__': spec} + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': ModuleSpec('', FakeLoader())} + with self.assertWarns(DeprecationWarning) as w: self.assertEqual(linecache.getlines(filename, module_globals), ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals; __loader__ != __spec__.loader') linecache.clearcache() spec = ModuleSpec('x.y.z', FakeLoader()) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e6666ddc638..a6af5057cc8 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -727,7 +727,7 @@ def check_module_globals(self, module_globals): def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): if self.module is py_warnings: - self.check_module_globals(module_globals) + self.check_module_globals_deprecated(module_globals, errmsg) return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') @@ -738,9 +738,6 @@ def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError) self.assertEqual(len(w), 0) def check_module_globals_deprecated(self, module_globals, msg): - if self.module is py_warnings: - self.check_module_globals(module_globals) - return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') self.module.warn_explicit( diff --git a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst new file mode 100644 index 00000000000..63e71c19f8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst @@ -0,0 +1,4 @@ +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). From 49e74210cb652d8bd538a4cc887f507396cfc893 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 15:50:03 +0100 Subject: [PATCH 190/417] gh-139344: Remove pending removal notice for undeprecated importlib.resources API (GH-141507) --- Doc/deprecations/pending-removal-in-3.13.rst | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a..d5b8c80e8f9 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) From 10bec7c1eb3ee27f490a067426eef452b15f78f9 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 14 Nov 2025 19:52:01 +0500 Subject: [PATCH 191/417] GH-141312: Allow only integers to longrangeiter_setstate state (GH-141317) This fixes an assertion error when the new computed start is not an integer. --- Lib/test/test_range.py | 10 ++++++++++ .../2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst | 2 ++ Objects/rangeobject.c | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 3870b153688..2c9c290e890 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -470,6 +470,16 @@ def test_iterator_setstate(self): it.__setstate__(2**64 - 7) self.assertEqual(list(it), [12, 10]) + def test_iterator_invalid_setstate(self): + for invalid_value in (1.0, ""): + ranges = (('rangeiter', range(10, 100, 2)), + ('longrangeiter', range(10, 2**65, 2))) + for rng_name, rng in ranges: + with self.subTest(invalid_value=invalid_value, range=rng_name): + it = iter(rng) + with self.assertRaises(TypeError): + it.__setstate__(invalid_value) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst new file mode 100644 index 00000000000..fdb136cef3f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst @@ -0,0 +1,2 @@ +Fix the assertion failure in the ``__setstate__`` method of the range iterator +when a non-integer argument is passed. Patch by Sergey Miryanov. diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index f8cdfe68a64..e93346fb277 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1042,6 +1042,11 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) static PyObject * longrangeiter_setstate(PyObject *op, PyObject *state) { + if (!PyLong_CheckExact(state)) { + PyErr_Format(PyExc_TypeError, "state must be an int, not %T", state); + return NULL; + } + longrangeiterobject *r = (longrangeiterobject*)op; PyObject *zero = _PyLong_GetZero(); // borrowed reference int cmp; From fa245df4a0848c15cf8d907c10fc92819994b866 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 14 Nov 2025 19:55:04 +0500 Subject: [PATCH 192/417] GH-141509: Fix warning about remaining subinterpreters (GH-141528) Co-authored-by: Peter Bierma --- Lib/test/test_interpreters/test_api.py | 2 +- .../2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst | 3 +++ Python/pylifecycle.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 9a5ee03e472..fd9e46bf335 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -432,7 +432,7 @@ def test_cleanup_in_repl(self): exit()""" stdout, stderr = repl.communicate(script) self.assertIsNone(stderr) - self.assertIn(b"remaining subinterpreters", stdout) + self.assertIn(b"Interpreter.close()", stdout) self.assertNotIn(b"Traceback", stdout) @support.requires_subprocess() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst new file mode 100644 index 00000000000..a51aa495228 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst @@ -0,0 +1,3 @@ +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of the +private ``_interpreters.destroy`` function when warning about remaining subinterpreters. +Patch by Sergey Miryanov. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 805805ef188..67368b5ce07 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2643,7 +2643,7 @@ finalize_subinterpreters(void) (void)PyErr_WarnEx( PyExc_RuntimeWarning, "remaining subinterpreters; " - "destroy them with _interpreters.destroy()", + "close them with Interpreter.close()", 0); /* Swap out the current tstate, which we know must belong From a415a1812c4d7798131d077c8776503bb3e1844f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Nov 2025 15:56:37 +0100 Subject: [PATCH 193/417] gh-139653: Remove assertions in _Py_InitializeRecursionLimits() (#141551) These checks were invalid and failed randomly on FreeBSD and Alpine Linux. --- Python/ceval.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b76c9ec2811..31b81a37464 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -523,13 +523,6 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; ts->c_stack_init_base = base; ts->c_stack_init_top = top; - - // Test the stack pointer -#if !defined(NDEBUG) && !defined(__wasi__) - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - assert(ts->c_stack_soft_limit < here_addr); - assert(here_addr < ts->c_stack_top); -#endif } From eab7385858025df9fcb0131f71ec4a46d44e3ae9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 16:05:42 +0100 Subject: [PATCH 194/417] gh-116146: Avoid empty braces in _testembed.c (GH-141556) --- Programs/_testembed.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 27224e508bd..d0d7d5f03fb 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2255,7 +2255,7 @@ exec_module(PyObject* self, PyObject* mod) static PyMethodDef create_static_module_methods[] = { {"create_module", create_module, METH_O, NULL}, {"exec_module", exec_module, METH_O, NULL}, - {} + {NULL} }; static struct PyModuleDef create_static_module_def = { From b101e9d36b1aed2bb4bca8aec3e1cc1d1df4f79e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 14 Nov 2025 15:23:01 +0000 Subject: [PATCH 195/417] Add PyManager troubleshooting steps for direct launch of script files (GH-141530) --- Doc/using/windows.rst | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index e6619b73bd2..ee182519199 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,6 +4,8 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* @@ -543,12 +545,9 @@ configuration option. The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. - + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. .. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage @@ -859,6 +858,17 @@ default). These scripts are separated for each runtime, and so you may need to add multiple paths. + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + .. _windows-embeddable: From da7f4e4b22020cfc6c5b5918756e454ef281848d Mon Sep 17 00:00:00 2001 From: Locked-chess-official <13140752715@163.com> Date: Fri, 14 Nov 2025 23:52:14 +0800 Subject: [PATCH 196/417] gh-141488: Add `Py_` prefix to Include/datetime.h macros (#141493) --- Include/datetime.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e..ed36e6e48c8 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -263,5 +263,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ From f26ed455d5582a7d66618acf2a93bc4b22a84b47 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 14 Nov 2025 23:17:59 +0530 Subject: [PATCH 197/417] gh-114203: skip locking if object is already locked by two-mutex critical section (#141476) --- ...-11-14-16-25-15.gh-issue-114203.n3tlQO.rst | 1 + .../test_critical_sections.c | 101 ++++++++++++++++++ Python/critical_section.c | 23 +++- 3 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst new file mode 100644 index 00000000000..883f9333cae --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst @@ -0,0 +1 @@ +Skip locking if object is already locked by two-mutex critical section. diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index e0ba37abcdd..e3b2fe716d4 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -284,10 +284,111 @@ test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) #endif +#ifdef Py_GIL_DISABLED + +static PyObject * +test_critical_section1_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + + PyCriticalSection cs1, cs2; + // First acquisition of critical section on object locks it + PyCriticalSection_Begin(&cs1, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Attempting to re-acquire critical section on same object which + // is already locked by top-most critical section is a no-op. + PyCriticalSection_Begin(&cs2, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing second critical section is a no-op. + PyCriticalSection_End(&cs2); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing first critical section unlocks the object + PyCriticalSection_End(&cs1); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + + Py_DECREF(a); + Py_RETURN_NONE; +} + +static PyObject * +test_critical_section2_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + PyObject *b = PyDict_New(); + assert(b != NULL); + + PyCriticalSection2 cs; + // First acquisition of critical section on objects locks them + PyCriticalSection2_Begin(&cs, a, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Attempting to re-acquire critical section on either of two + // objects already locked by top-most critical section is a no-op. + + // Check re-acquiring on first object + PyCriticalSection a_cs; + PyCriticalSection_Begin(&a_cs, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&a_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Check re-acquiring on second object + PyCriticalSection b_cs; + PyCriticalSection_Begin(&b_cs, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&b_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Releasing critical section on both objects unlocks them + PyCriticalSection2_End(&cs); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + assert(!PyMutex_IsLocked(&b->ob_mutex)); + + Py_DECREF(a); + Py_DECREF(b); + Py_RETURN_NONE; +} + +#endif // Py_GIL_DISABLED + static PyMethodDef test_methods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, +#ifdef Py_GIL_DISABLED + {"test_critical_section1_reacquisition", test_critical_section1_reacquisition, METH_NOARGS}, + {"test_critical_section2_reacquisition", test_critical_section2_reacquisition, METH_NOARGS}, +#endif #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, diff --git a/Python/critical_section.c b/Python/critical_section.c index e628ba2f6d1..218b580e951 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -24,11 +24,24 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) // As an optimisation for locking the same object recursively, skip // locking if the mutex is currently locked by the top-most critical // section. - if (tstate->critical_section && - untag_critical_section(tstate->critical_section)->_cs_mutex == m) { - c->_cs_mutex = NULL; - c->_cs_prev = 0; - return; + // If the top-most critical section is a two-mutex critical section, + // then locking is skipped if either mutex is m. + if (tstate->critical_section) { + PyCriticalSection *prev = untag_critical_section(tstate->critical_section); + if (prev->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + if (tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + if (prev2->_cs_mutex2 == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + } } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; From 1281be1caf9357ee2a68f7370a88b5cff0110e15 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Sat, 15 Nov 2025 00:38:39 +0300 Subject: [PATCH 198/417] gh-141367: Use CALL_LIST_APPEND instruction only for lists, not for list subclasses (GH-141398) Co-authored-by: Ken Jin --- Include/internal/pycore_code.h | 4 +-- Lib/test/test_opcache.py | 27 +++++++++++++++++++ ...-11-11-13-40-45.gh-issue-141367.I5KY7F.rst | 2 ++ Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 4 --- Python/generated_cases.c.h | 7 +---- Python/specialize.c | 17 +++++++----- 7 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 9748e036bf2..cb9c0aa27a1 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -311,8 +311,8 @@ PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins _Py_CODEUNIT *instr, PyObject *name); PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index f23f8c053e8..c7eea75117d 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1872,6 +1872,33 @@ def for_iter_generator(): self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") self.assert_no_opcode(for_iter_generator, "FOR_ITER") + @cpython_only + @requires_specialization_ft + def test_call_list_append(self): + # gh-141367: only exact lists should use + # CALL_LIST_APPEND instruction after specialization. + + r = range(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + def list_append(l): + for _ in r: + l.append(1) + + list_append([]) + self.assert_specialized(list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(list_append, "CALL") + + def my_list_append(l): + for _ in r: + l.append(1) + + class MyList(list): pass + my_list_append(MyList()) + self.assert_specialized(my_list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(my_list_append, "CALL") + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst new file mode 100644 index 00000000000..cb830fcd9e1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst @@ -0,0 +1,2 @@ +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2c798855a71..8a7b784bb9e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3689,7 +3689,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -4395,7 +4395,6 @@ dummy_func( assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - DEOPT_IF(!PyList_CheckExact(self_o)); DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7ba2e9d0d92..6796abf84ac 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6037,10 +6037,6 @@ callable = stack_pointer[-3]; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a984da6dc91..01f65d9dd37 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1533,7 +1533,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -3470,11 +3470,6 @@ self = nos; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); diff --git a/Python/specialize.c b/Python/specialize.c index 2193596a331..19433bc7a74 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1602,8 +1602,8 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } static int -specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, - int nargs) +specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, + _Py_CODEUNIT *instr, int nargs) { switch (descr->d_method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | @@ -1627,8 +1627,11 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - specialize(instr, CALL_LIST_APPEND); - return 0; + assert(self_or_null != NULL); + if (PyList_CheckExact(self_or_null)) { + specialize(instr, CALL_LIST_APPEND); + return 0; + } } specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; @@ -1766,7 +1769,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } Py_NO_INLINE void -_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); @@ -1784,7 +1787,9 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) fail = specialize_class_call(callable, instr, nargs); } else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { - fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs); + PyObject *self_or_null = PyStackRef_AsPyObjectBorrow(self_or_null_st); + fail = specialize_method_descriptor((PyMethodDescrObject *)callable, + self_or_null, instr, nargs); } else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; From f0a8bc737ab2f04d4196eee154cb1e17e26ad585 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Nov 2025 17:25:45 -0600 Subject: [PATCH 199/417] gh-140938: Raise ValueError for infinite inputs to stdev/pstdev (GH-141531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise ValueError for infinite inputs to stdev/pstdev --- Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/statistics.py | 18 ++++++++++++++---- Lib/test/test_statistics.py | 9 ++++++++- ...5-11-13-14-51-30.gh-issue-140938.kXsHHv.rst | 2 ++ 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst diff --git a/Lib/statistics.py b/Lib/statistics.py index 3d805cb0739..26cf925529e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -619,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -637,9 +642,14 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) ## Statistics for relations between two inputs ############################# diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 8250b0aef09..677a87b51b9 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2005,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2113,6 +2112,14 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + class TestSqrtHelpers(unittest.TestCase): def test_integer_sqrt_of_frac_rto(self): diff --git a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst new file mode 100644 index 00000000000..bd3044002a2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst @@ -0,0 +1,2 @@ +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now raise a +:exc:`ValueError` when the input contains an infinity or a NaN. From 453d886f8592d2f4346d5621b1e4ff31c24338d5 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Fri, 14 Nov 2025 19:13:37 -0500 Subject: [PATCH 200/417] GH-90344: replace single-call `io.IncrementalNewlineDecoder` usage with non-incremental newline decoders (GH-30276) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Brett Cannon --- Lib/doctest.py | 9 ++------- Lib/importlib/_bootstrap_external.py | 3 +-- Lib/test/test_importlib/test_abc.py | 2 +- .../2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst | 1 + 4 files changed, 5 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index 92a2ab4f7e6..ad8fb900f69 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,7 +104,7 @@ def _test(): import traceback import types import unittest -from io import StringIO, IncrementalNewlineDecoder +from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple import _colorize # Used in doctests from _colorize import ANSIColors, can_colorize @@ -237,10 +237,6 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") -def _newline_convert(data): - # The IO module provides a handy decoder for universal newline conversion - return IncrementalNewlineDecoder(None, True).decode(data, True) - def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -252,10 +248,9 @@ def _load_testfile(filename, package, module_relative, encoding): pass if hasattr(loader, 'get_data'): file_contents = loader.get_data(filename) - file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + return TextIOWrapper(BytesIO(file_contents), encoding=encoding, newline=None).read(), filename with open(filename, encoding=encoding) as f: return f.read(), filename diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4ab0e79ea6e..192c0261408 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -552,8 +552,7 @@ def decode_source(source_bytes): import tokenize # To avoid bootstrap issues. source_bytes_readline = _io.BytesIO(source_bytes).readline encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) + return _io.TextIOWrapper(_io.BytesIO(source_bytes), encoding=encoding[0], newline=None).read() # Module specifications ####################################################### diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index dd943210ffc..bd1540ce403 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -904,7 +904,7 @@ def test_universal_newlines(self): mock = self.SourceOnlyLoaderMock('mod.file') source = "x = 42\r\ny = -13\r\n" mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) + expect = io.StringIO(source, newline=None).getvalue() self.assertEqual(mock.get_source(name), expect) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst new file mode 100644 index 00000000000..b1d05354f65 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst @@ -0,0 +1 @@ +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was being called once. From 53d65c840e038ce9a5782fbd3da963c7aba90570 Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Fri, 14 Nov 2025 16:59:51 -0800 Subject: [PATCH 201/417] gh-136442: Fix unittest to return exit code 5 when setUpClass raises an exception (#136487) --- Lib/test/test_unittest/test_program.py | 20 +++++++++++++++++++ Lib/unittest/main.py | 10 +++++----- ...-07-09-21-45-51.gh-issue-136442.jlbklP.rst | 1 + 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 6092ed292d8..8ed92373e5e 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -75,6 +75,14 @@ def testUnexpectedSuccess(self): class Empty(unittest.TestCase): pass + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing the supplied testcase.""" @@ -191,6 +199,18 @@ def test_ExitEmptySuite(self): out = stream.getvalue() self.assertIn('\nNO TESTS RAN\n', out) + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) + class InitialisableProgram(unittest.TestProgram): exit = False diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6fd949581f3..be99d93c78c 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,12 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0 and len(self.result.skipped) == 0: - sys.exit(_NO_TESTS_EXITCODE) - elif self.result.wasSuccessful(): - sys.exit(0) - else: + if not self.result.wasSuccessful(): sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: + sys.exit(_NO_TESTS_EXITCODE) + else: + sys.exit(0) main = TestProgram diff --git a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst new file mode 100644 index 00000000000..f87fb1113ca --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst @@ -0,0 +1 @@ +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` raises an exception From 4ceb077c5cea30fef734f4c4e92c18d978be6c38 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 15 Nov 2025 02:23:54 +0000 Subject: [PATCH 202/417] gh-141579: Fix perf_jit backend in sys.activate_stack_trampoline() (#141580) --- Lib/test/test_perf_profiler.py | 18 ++++++++++++++++++ ...5-11-15-01-21-00.gh-issue-141579.aB7cD9.rst | 2 ++ Python/sysmodule.c | 16 ++++++++-------- 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 13424991639..e6852c93e69 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -238,6 +238,24 @@ def test_sys_api_get_status(self): """ assert_python_ok("-c", code, PYTHON_JIT="0") + def test_sys_api_perf_jit_backend(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + assert sys.is_stack_trampoline_active() is True + sys.deactivate_stack_trampoline() + assert sys.is_stack_trampoline_active() is False + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + + def test_sys_api_with_existing_perf_jit_trampoline(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + sys.activate_stack_trampoline("perf_jit") + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst new file mode 100644 index 00000000000..8ab9979c399 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst @@ -0,0 +1,2 @@ +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a611844f76e..b4b441bf4d9 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2380,14 +2380,14 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } - else if (strcmp(backend, "perf_jit") == 0) { - _PyPerf_Callbacks cur_cb; - _PyPerfTrampoline_GetCallbacks(&cur_cb); - if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { - PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); - return NULL; - } + } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; } } } From ed81baf81f144e14510c492b71cf860472b0a0b7 Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Sun, 16 Nov 2025 00:14:23 +0800 Subject: [PATCH 203/417] gh-140458: `xmlrpc.client` raises Fault, does not returns it. (GH-140759) --- Doc/library/xmlrpc.client.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index a21c7d3e4e3..e4912629aac 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -179,9 +179,9 @@ ServerProxy Objects A :class:`ServerProxy` instance has a method corresponding to each remote procedure call accepted by the XML-RPC server. Calling the method performs an RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. +can be overloaded with multiple argument signatures). The RPC finishes either +by returning data in a conformant type or by raising a :class:`Fault` or +:class:`ProtocolError` exception indicating an error. Servers that support the XML introspection API support some common methods grouped under the reserved :attr:`~ServerProxy.system` attribute: From 85f3009d7504ddcc01de715c494067e89c16303c Mon Sep 17 00:00:00 2001 From: Shamil Date: Sat, 15 Nov 2025 20:46:54 +0300 Subject: [PATCH 204/417] gh-141553: Fix incorrect function signatures in `_testmultiphase` (#141554) --- Modules/_testmultiphase.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 220fa888e49..cd2d7b65598 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -1061,7 +1061,7 @@ PyModInit__test_from_modexport_exception(void) } static PyObject * -modexport_create_string(PyObject *spec, PyObject *def) +modexport_create_string(PyObject *spec, PyModuleDef *def) { assert(def == NULL); return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); @@ -1138,8 +1138,9 @@ modexport_get_empty_slots(PyObject *mod, PyObject *arg) } static void -modexport_smoke_free(PyObject *mod) +modexport_smoke_free(void *op) { + PyObject *mod = (PyObject *)op; int *state = PyModule_GetState(mod); if (!state) { PyErr_FormatUnraisable("Exception ignored in module %R free", mod); From ed73c909f278a1eb558b120ef8ed2c0f8528bf58 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 16 Nov 2025 04:19:41 +0800 Subject: [PATCH 205/417] gh-139109: JIT _EXIT_TRACE to ENTER_EXECUTOR rather than _DEOPT (GH-141573) --- Include/internal/pycore_optimizer.h | 2 +- Lib/test/test_capi/test_opt.py | 36 +++++++++++++++++++++++++++++ Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- Python/optimizer.c | 6 ++--- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 653285a2c6b..0307a174e77 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f06c6cbda29..25372fee58e 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -40,6 +40,17 @@ def get_first_executor(func): pass return None +def get_all_executors(func): + code = func.__code__ + co_code = code.co_code + executors = [] + for i in range(0, len(co_code), 2): + try: + executors.append(_opcode.get_executor(code, i)) + except ValueError: + pass + return executors + def iter_opnames(ex): for item in ex: @@ -2629,6 +2640,31 @@ def gen(): next(g) """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) + def test_executor_side_exits_create_another_executor(self): + def f(): + for x in range(TIER2_THRESHOLD + 3): + for y in range(TIER2_THRESHOLD + 3): + z = x + y + + f() + all_executors = get_all_executors(f) + # Inner loop warms up first. + # Outer loop warms up later, linking to the inner one. + # Therefore, we have at least two executors. + self.assertGreaterEqual(len(all_executors), 2) + for executor in all_executors: + opnames = list(get_opnames(executor)) + # Assert all executors first terminator ends in + # _EXIT_TRACE or _JUMP_TO_TOP, not _DEOPT + for idx, op in enumerate(opnames): + if op == "_EXIT_TRACE" or op == "_JUMP_TO_TOP": + break + elif op == "_DEOPT": + self.fail(f"_DEOPT encountered first at executor" + f" {executor} at offset {idx} rather" + f" than expected _EXIT_TRACE") + + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a7b784bb9e..565eaa7a599 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5643,7 +5643,7 @@ dummy_func( bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); if (full) { LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); @@ -5683,7 +5683,7 @@ dummy_func( #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); ERROR_IF(err < 0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 01f65d9dd37..0d4678df68c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12263,7 +12263,7 @@ JUMP_TO_LABEL(error); opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (full) { LEAVE_TRACING(); @@ -12309,7 +12309,7 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); stack_pointer = _PyFrame_GetStackPointer(frame); LEAVE_TRACING(); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/optimizer.c b/Python/optimizer.c index 65007a256d0..9db894f0bf0 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -574,7 +574,7 @@ _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, - bool stop_tracing) + int stop_tracing_opcode) { #ifdef Py_DEBUG @@ -637,8 +637,8 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (stop_tracing) { - ADD_TO_TRACE(_DEOPT, 0, 0, target); + if (stop_tracing_opcode != 0) { + ADD_TO_TRACE(stop_tracing_opcode, 0, 0, target); goto done; } From e33afa7ddbca3fca38f4ec4369b620c37cb092e2 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:50:54 +0000 Subject: [PATCH 206/417] gh-141004: Document the `PyPickleBuffer_*` C API (GH-141630) Co-authored-by: Peter Bierma --- Doc/c-api/concrete.rst | 1 + Doc/c-api/picklebuffer.rst | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 Doc/c-api/picklebuffer.rst diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index a5c5a53236c..1746fe95eaa 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,6 +109,7 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst frame.rst diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 00000000000..9e2d92341b0 --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. From 5348c200f5b26d6dd21d900b2b4cb684150d4b01 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 16 Nov 2025 10:58:28 -0800 Subject: [PATCH 207/417] gh-125115 : Refactor the pdb parsing issue so positional arguments can pass through (#140933) --- Lib/pdb.py | 86 ++++++++++--------- Lib/test/test_pdb.py | 5 +- ...-11-03-05-38-31.gh-issue-125115.jGS8MN.rst | 1 + 3 files changed, 52 insertions(+), 40 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index b799a113503..76bb28d7396 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3548,7 +3548,15 @@ def exit_with_permission_help_text(): sys.exit(1) -def main(): +def parse_args(): + # We want pdb to be as intuitive as possible to users, so we need to do some + # heuristic parsing to deal with ambiguity. + # For example: + # "python -m pdb -m foo -p 1" should pass "-p 1" to "foo". + # "python -m pdb foo.py -m bar" should pass "-m bar" to "foo.py". + # "python -m pdb -m foo -m bar" should pass "-m bar" to "foo". + # This require some customized parsing logic to find the actual debug target. + import argparse parser = argparse.ArgumentParser( @@ -3559,28 +3567,48 @@ def main(): color=True, ) - # We need to maunally get the script from args, because the first positional - # arguments could be either the script we need to debug, or the argument - # to the -m module + # Get all the commands out first. For backwards compatibility, we allow + # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', help='pdb commands to execute as if given in a .pdbrc file') - parser.add_argument('-m', metavar='module', dest='module') - parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) - - if len(sys.argv) == 1: - # If no arguments were given (python -m pdb), print the whole help message. - # Without this check, argparse would only complain about missing required arguments. - parser.print_help() - sys.exit(2) opts, args = parser.parse_known_args() - if opts.pid: - # If attaching to a remote pid, unrecognized arguments are not allowed. - # This will raise an error if there are extra unrecognized arguments. - opts = parser.parse_args() - if opts.module: - parser.error("argument -m: not allowed with argument --pid") + if not args: + # If no arguments were given (python -m pdb), print the whole help message. + # Without this check, argparse would only complain about missing required arguments. + # We need to add the arguments definitions here to get a proper help message. + parser.add_argument('-m', metavar='module', dest='module') + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) + parser.print_help() + sys.exit(2) + elif args[0] == '-p' or args[0] == '--pid': + # Attach to a pid + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) + opts, args = parser.parse_known_args() + if args: + # For --pid, any extra arguments are invalid. + parser.error(f"unrecognized arguments: {' '.join(args)}") + elif args[0] == '-m': + # Debug a module, we only need the first -m module argument. + # The rest is passed to the module itself. + parser.add_argument('-m', metavar='module', dest='module') + opt_module = parser.parse_args(args[:2]) + opts.module = opt_module.module + args = args[2:] + elif args[0].startswith('-'): + # Invalid argument before the script name. + invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) + parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") + + # Otherwise it's debugging a script and we already parsed all -c commands. + + return opts, args + +def main(): + opts, args = parse_args() + + if getattr(opts, 'pid', None) is not None: try: attach(opts.pid, opts.commands) except RuntimeError: @@ -3592,30 +3620,10 @@ def main(): except PermissionError: exit_with_permission_help_text() return - elif opts.module: - # If a module is being debugged, we consider the arguments after "-m module" to - # be potential arguments to the module itself. We need to parse the arguments - # before "-m" to check if there is any invalid argument. - # e.g. "python -m pdb -m foo --spam" means passing "--spam" to "foo" - # "python -m pdb --spam -m foo" means passing "--spam" to "pdb" and is invalid - idx = sys.argv.index('-m') - args_to_pdb = sys.argv[1:idx] - # This will raise an error if there are invalid arguments - parser.parse_args(args_to_pdb) - else: - # If a script is being debugged, then pdb expects the script name as the first argument. - # Anything before the script is considered an argument to pdb itself, which would - # be invalid because it's not parsed by argparse. - invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) - if invalid_args: - parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - - if opts.module: + elif getattr(opts, 'module', None) is not None: file = opts.module target = _ModuleTarget(file) else: - if not args: - parser.error("no module or script to run") file = args.pop(0) if file.endswith('.pyz'): target = _ZipTarget(file) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 9a7d8550035..2ca689e0adf 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3974,7 +3974,10 @@ def test_run_module_with_args(self): commands = """ continue """ - self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + self._run_pdb(["calendar", "-m"], commands, expected_returncode=1) + + _, stderr = self._run_pdb(["-m", "calendar", "-p", "1"], commands) + self.assertIn("unrecognized arguments: -p", stderr) stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) self.assertIn("December", stdout) diff --git a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst new file mode 100644 index 00000000000..d36debec3ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst @@ -0,0 +1 @@ +Refactor the :mod:`pdb` parsing issue so positional arguments can pass through intuitively. From be699d6c7c8793d3eb464f2e5d3f10262fe3bc37 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 16 Nov 2025 14:25:50 -0500 Subject: [PATCH 208/417] gh-141004: Document missing `PyCFunction*` and `PyCMethod*` APIs (GH-141253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/c-api/structures.rst | 93 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 58dd915e04f..414dfdc84e6 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -447,6 +447,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -472,6 +491,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -482,6 +519,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- From 8be3b2f479431f670f2e81e41b52e698c0806289 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 16 Nov 2025 13:57:07 -0800 Subject: [PATCH 209/417] gh-136057: Allow step and next to step over for loops (#136160) --- Lib/bdb.py | 22 ++++++++++--- Lib/test/test_pdb.py | 31 +++++++++++++++++++ ...-07-01-04-57-57.gh-issue-136057.4-t596.rst | 1 + 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235a..50cf2b3f5b3 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -297,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -526,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -539,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -564,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -572,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2ca689e0adf..9d89008756a 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3232,6 +3232,37 @@ def test_pdb_issue_gh_127321(): """ +def test_pdb_issue_gh_136057(): + """See GH-136057 + "step" and "next" commands should be able to get over list comprehensions + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... lst = [i for i in range(10)] + ... for i in lst: pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'next', + ... 'next', + ... 'step', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) next + > (3)test_function() + -> lst = [i for i in range(10)] + (Pdb) next + > (4)test_function() + -> for i in lst: pass + (Pdb) step + --Return-- + > (4)test_function()->None + -> for i in lst: pass + (Pdb) continue + """ + + def test_pdb_issue_gh_80731(): """See GH-80731 diff --git a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst new file mode 100644 index 00000000000..e237a0e98cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst @@ -0,0 +1 @@ +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't go over the line if a loop exists in the line. From 7800b78067162fc9d7cb6926f703fe14dee1702a Mon Sep 17 00:00:00 2001 From: SubbaraoGarlapati <53627478+SubbaraoGarlapati@users.noreply.github.com> Date: Mon, 17 Nov 2025 06:23:12 -0500 Subject: [PATCH 210/417] fix memory order of `_Py_atomic_store_uint_release` (#141562) --- Include/cpython/pyatomic_std.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615e..7176f667a40 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { From 31ea3f3c76b33e8e3cc098721266fe17f459e75d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:32:00 +0000 Subject: [PATCH 211/417] gh-141018: Update `.exe`, `.dll`, `.rtf` and `.jpg` mime types in `mimetypes` (#141023) --- Lib/mimetypes.py | 8 +++---- Lib/test/test_mimetypes.py | 24 ++++++++----------- ...-11-04-20-08-41.gh-issue-141018.d_oyOI.rst | 2 ++ 3 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 42477713c78..07ac079186f 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -489,8 +489,6 @@ def _default_mime_types(): '.cjs' : 'application/node', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', - '.dll' : 'application/octet-stream', - '.exe' : 'application/octet-stream', '.o' : 'application/octet-stream', '.obj' : 'application/octet-stream', '.so' : 'application/octet-stream', @@ -501,12 +499,15 @@ def _default_mime_types(): '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', '.eps' : 'application/postscript', + '.rtf' : 'application/rtf', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', '.toml' : 'application/toml', '.trig' : 'application/trig', '.m3u' : 'application/vnd.apple.mpegurl', '.m3u8' : 'application/vnd.apple.mpegurl', + '.dll' : 'application/vnd.microsoft.portable-executable', + '.exe' : 'application/vnd.microsoft.portable-executable', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', '.eot' : 'application/vnd.ms-fontobject', @@ -649,7 +650,6 @@ def _default_mime_types(): '.pl' : 'text/plain', '.srt' : 'text/plain', '.rtx' : 'text/richtext', - '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', @@ -682,11 +682,9 @@ def _default_mime_types(): # Please sort these too common_types = _common_types_default = { - '.rtf' : 'application/rtf', '.apk' : 'application/vnd.android.package-archive', '.midi': 'audio/midi', '.mid' : 'audio/midi', - '.jpg' : 'image/jpg', '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 73414498359..0f29640bc1c 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -112,13 +112,12 @@ def test_non_standard_types(self): eq = self.assertEqual # First try strict eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + eq(self.db.guess_extension('image/jpeg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPEG', strict=False), '.jpg') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -179,8 +178,8 @@ def test_guess_all_types(self): self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) self.assertEqual(len(set(all)), len(all)) # no duplicates # And now non-strict - all = self.db.guess_all_extensions('image/jpg', strict=False) - self.assertEqual(all, ['.jpg']) + all = self.db.guess_all_extensions('image/jpeg', strict=False) + self.assertEqual(all, ['.jpg', '.jpe', '.jpeg']) # And now for no hits all = self.db.guess_all_extensions('image/jpg', strict=True) self.assertEqual(all, []) @@ -231,6 +230,7 @@ def check_extensions(): ("application/ogg", ".ogx"), ("application/pdf", ".pdf"), ("application/postscript", ".ps"), + ("application/rtf", ".rtf"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), ("application/vnd.apple.mpegurl", ".m3u"), @@ -281,7 +281,6 @@ def check_extensions(): ("model/stl", ".stl"), ("text/html", ".html"), ("text/plain", ".txt"), - ("text/rtf", ".rtf"), ("text/x-rst", ".rst"), ("video/matroska", ".mkv"), ("video/matroska-3d", ".mk3d"), @@ -372,9 +371,7 @@ def test_keywords_args_api(self): self.assertEqual(self.db.guess_type( url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( - type='image/jpg', strict=True), []) - self.assertEqual(self.db.guess_extension( - type='image/jpg', strict=False), '.jpg') + type='image/jpeg', strict=True), ['.jpg', '.jpe', '.jpeg']) def test_added_types_are_used(self): mimetypes.add_type('testing/default-type', '') @@ -452,15 +449,15 @@ def test_parse_args(self): args, help_text = mimetypes._parse_args("--invalid") self.assertTrue(help_text.startswith("usage: ")) - args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpeg")) self.assertTrue(args.extension) self.assertTrue(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) - args, _ = mimetypes._parse_args(shlex.split("-e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-e image/jpeg")) self.assertTrue(args.extension) self.assertFalse(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) args, _ = mimetypes._parse_args(shlex.split("-l foo.webp")) self.assertFalse(args.extension) @@ -491,7 +488,6 @@ def test_multiple_inputs_error(self): def test_invocation(self): for command, expected in [ - ("-l -e image/jpg", ".jpg"), ("-e image/jpeg", ".jpg"), ("-l foo.webp", "type: image/webp encoding: None"), ]: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst new file mode 100644 index 00000000000..e776515a9fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst @@ -0,0 +1,2 @@ +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. From df8091d516f874bd8222569794229ea77fb3a0a3 Mon Sep 17 00:00:00 2001 From: Tamzin Hadasa Kelly Date: Mon, 17 Nov 2025 18:35:01 +0700 Subject: [PATCH 212/417] gh-141650: Fix typo in `xml.sax.saxutils.unescape` documentation (#141652) --- Doc/library/xml.sax.utils.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 5ee11d58c3d..7731f03d875 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -37,7 +37,7 @@ or as base classes. You can unescape other strings of data by passing a dictionary as the optional *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` + replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` are always unescaped, even if *entities* is provided. From d527d3bf8beb9cd26c179f2c0111d635cdaa9cd3 Mon Sep 17 00:00:00 2001 From: dereckduran <67027239+dereckduran@users.noreply.github.com> Date: Mon, 17 Nov 2025 03:44:44 -0800 Subject: [PATCH 213/417] gh-62480: De-personalize "Coping with mutable arguments" section in `unittest.mock` examples (#141323) --- Doc/library/unittest.mock-examples.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 6af4298d44f..61c75b5a03b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -863,9 +863,9 @@ Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the +example we're using *another* mock to store the arguments so that we can use the mock methods for doing the assertion. Again a helper function sets this up for -me. :: +us. :: >>> from copy import deepcopy >>> from unittest.mock import Mock, patch, DEFAULT From 20b64bdf23b88e44f72bc49f8bc783ae8ca21511 Mon Sep 17 00:00:00 2001 From: Thomas Ballard Date: Mon, 17 Nov 2025 06:47:28 -0500 Subject: [PATCH 214/417] Docs: Fix typo in socketserver documentation (#140956) --- Doc/library/socketserver.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7bc2f7afbbb..491b8769f44 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -546,7 +546,7 @@ The difference is that the ``readline()`` call in the second handler will call first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. From 994ab5c922b179ab1884f05b3440c24db9e9733d Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 17 Nov 2025 20:43:14 +0800 Subject: [PATCH 215/417] gh-140729: Add __mp_main__ as a duplicate for __main__ for pickle to work (#140735) --- Lib/profiling/sampling/_sync_coordinator.py | 11 +++- .../test_profiling/test_sampling_profiler.py | 52 ++++++++++++++++++- ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 + 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index adb040e89cc..be63dbe3e90 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -10,6 +10,7 @@ import socket import runpy import time +import types from typing import List, NoReturn @@ -175,15 +176,21 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: with open(script_path, 'rb') as f: source_code = f.read() + except FileNotFoundError as e: raise TargetError(f"Script file not found: {script_path}") from e except PermissionError as e: raise TargetError(f"Permission denied reading script: {script_path}") from e try: - # Compile and execute the script + main_module = types.ModuleType("__main__") + main_module.__file__ = script_path + main_module.__builtins__ = __builtins__ + # gh-140729: Create a __mp_main__ module to allow pickling + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + code = compile(source_code, script_path, 'exec', module='__main__') - exec(code, {'__name__': '__main__', '__file__': script_path}) + exec(code, main_module.__dict__) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e except SystemExit: diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 5b924cb2453..0ba6799a1ce 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -22,7 +22,13 @@ from profiling.sampling.gecko_collector import GeckoCollector from test.support.os_helper import unlink -from test.support import force_not_colorized_test_class, SHORT_TIMEOUT +from test.support import ( + force_not_colorized_test_class, + SHORT_TIMEOUT, + script_helper, + os_helper, + SuppressCrashReport, +) from test.support.socket_helper import find_unused_port from test.support import requires_subprocess, is_emscripten from test.support import captured_stdout, captured_stderr @@ -3009,5 +3015,49 @@ def test_parse_mode_function(self): profiling.sampling.sample._parse_mode("invalid") +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = ''' +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +''' + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, 'test_process_pool_executor_pickle', test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", "profiling.sampling.sample", + "-d", "5", + "-i", "100000", + script, + stderr=subprocess.PIPE, + text=True + ) as proc: + proc.wait(timeout=SHORT_TIMEOUT) + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + if "PermissionError" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst new file mode 100644 index 00000000000..6725547667f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst @@ -0,0 +1,2 @@ +Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` +script can not be properly pickled and executed in worker processes. From 89a914c58db1661cb9da4f3b9e52c20bb4b02287 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 17 Nov 2025 12:46:26 +0000 Subject: [PATCH 216/417] gh-135953: Add GIL contention markers to sampling profiler Gecko format (#139485) This commit enhances the Gecko format reporter in the sampling profiler to include markers for GIL acquisition events. --- Include/cpython/pystate.h | 3 + Include/internal/pycore_debug_offsets.h | 4 + Lib/profiling/sampling/collector.py | 31 +-- Lib/profiling/sampling/gecko_collector.py | 237 ++++++++++++++++-- Lib/profiling/sampling/sample.py | 37 ++- Lib/test/test_external_inspection.py | 154 +++++++++++- .../test_profiling/test_sampling_profiler.py | 116 ++++++++- Modules/_remote_debugging_module.c | 123 +++++++-- Python/ceval_gil.c | 4 + 9 files changed, 627 insertions(+), 82 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c53abe43ebe..1e1e46ea4c0 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -113,6 +113,9 @@ struct _ts { /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 8e7cd16acff..f6d50bf5df7 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -106,6 +106,8 @@ typedef struct _Py_DebugOffsets { uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; } thread_state; // InterpreterFrame offset; @@ -273,6 +275,8 @@ typedef struct _Py_DebugOffsets { .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index b7a033ac0a6..3c2325ef772 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -1,17 +1,14 @@ from abc import ABC, abstractmethod -# Enums are slow -THREAD_STATE_RUNNING = 0 -THREAD_STATE_IDLE = 1 -THREAD_STATE_GIL_WAIT = 2 -THREAD_STATE_UNKNOWN = 3 - -STATUS = { - THREAD_STATE_RUNNING: "running", - THREAD_STATE_IDLE: "idle", - THREAD_STATE_GIL_WAIT: "gil_wait", - THREAD_STATE_UNKNOWN: "unknown", -} +# Thread status flags +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback for tests or when module is not available + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) class Collector(ABC): @abstractmethod @@ -26,8 +23,14 @@ def _iter_all_frames(self, stack_frames, skip_idle=False): """Iterate over all frame stacks from all interpreters and threads.""" for interpreter_info in stack_frames: for thread_info in interpreter_info.threads: - if skip_idle and thread_info.status != THREAD_STATE_RUNNING: - continue + # skip_idle now means: skip if thread is not actively running + # A thread is "active" if it has the GIL OR is on CPU + if skip_idle: + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + if not (has_gil or on_cpu): + continue frames = thread_info.frame_info if frames: yield frames, thread_info.thread_id diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 548acbf24b7..6c6700f1130 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -1,9 +1,20 @@ +import itertools import json import os import platform +import sys +import threading import time -from .collector import Collector, THREAD_STATE_RUNNING +from .collector import Collector +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback if module not available (shouldn't happen in normal use) + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) # Categories matching Firefox Profiler expectations @@ -11,14 +22,20 @@ {"name": "Other", "color": "grey", "subcategories": ["Other"]}, {"name": "Python", "color": "yellow", "subcategories": ["Other"]}, {"name": "Native", "color": "blue", "subcategories": ["Other"]}, - {"name": "Idle", "color": "transparent", "subcategories": ["Other"]}, + {"name": "GC", "color": "orange", "subcategories": ["Other"]}, + {"name": "GIL", "color": "green", "subcategories": ["Other"]}, + {"name": "CPU", "color": "purple", "subcategories": ["Other"]}, + {"name": "Code Type", "color": "red", "subcategories": ["Other"]}, ] # Category indices CATEGORY_OTHER = 0 CATEGORY_PYTHON = 1 CATEGORY_NATIVE = 2 -CATEGORY_IDLE = 3 +CATEGORY_GC = 3 +CATEGORY_GIL = 4 +CATEGORY_CPU = 5 +CATEGORY_CODE_TYPE = 6 # Subcategory indices DEFAULT_SUBCATEGORY = 0 @@ -58,6 +75,56 @@ def __init__(self, *, skip_idle=False): self.last_sample_time = 0 self.interval = 1.0 # Will be calculated from actual sampling + # State tracking for interval markers (tid -> start_time) + self.has_gil_start = {} # Thread has the GIL + self.no_gil_start = {} # Thread doesn't have the GIL + self.on_cpu_start = {} # Thread is running on CPU + self.off_cpu_start = {} # Thread is off CPU + self.python_code_start = {} # Thread running Python code (has GIL) + self.native_code_start = {} # Thread running native code (on CPU without GIL) + self.gil_wait_start = {} # Thread waiting for GIL + + # GC event tracking: track GC start time per thread + self.gc_start_per_thread = {} # tid -> start_time + + # Track which threads have been initialized for state tracking + self.initialized_threads = set() + + def _track_state_transition(self, tid, condition, active_dict, inactive_dict, + active_name, inactive_name, category, current_time): + """Track binary state transitions and emit markers. + + Args: + tid: Thread ID + condition: Whether the active state is true + active_dict: Dict tracking start time of active state + inactive_dict: Dict tracking start time of inactive state + active_name: Name for active state marker + inactive_name: Name for inactive state marker + category: Gecko category for the markers + current_time: Current timestamp + """ + # On first observation of a thread, just record the current state + # without creating a marker (we don't know what the previous state was) + if tid not in self.initialized_threads: + if condition: + active_dict[tid] = current_time + else: + inactive_dict[tid] = current_time + return + + # For already-initialized threads, track transitions + if condition: + active_dict.setdefault(tid, current_time) + if tid in inactive_dict: + self._add_marker(tid, inactive_name, inactive_dict.pop(tid), + current_time, category) + else: + inactive_dict.setdefault(tid, current_time) + if tid in active_dict: + self._add_marker(tid, active_name, active_dict.pop(tid), + current_time, category) + def collect(self, stack_frames): """Collect a sample from stack frames.""" current_time = (time.time() * 1000) - self.start_time @@ -69,19 +136,12 @@ def collect(self, stack_frames): ) / self.sample_count self.last_sample_time = current_time + # Process threads and track GC per thread for interpreter_info in stack_frames: for thread_info in interpreter_info.threads: - if ( - self.skip_idle - and thread_info.status != THREAD_STATE_RUNNING - ): - continue - frames = thread_info.frame_info - if not frames: - continue - tid = thread_info.thread_id + gc_collecting = thread_info.gc_collecting # Initialize thread if needed if tid not in self.threads: @@ -89,6 +149,80 @@ def collect(self, stack_frames): thread_data = self.threads[tid] + # Decode status flags + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + gil_requested = bool(status_flags & THREAD_STATUS_GIL_REQUESTED) + + # Track GIL possession (Has GIL / No GIL) + self._track_state_transition( + tid, has_gil, self.has_gil_start, self.no_gil_start, + "Has GIL", "No GIL", CATEGORY_GIL, current_time + ) + + # Track CPU state (On CPU / Off CPU) + self._track_state_transition( + tid, on_cpu, self.on_cpu_start, self.off_cpu_start, + "On CPU", "Off CPU", CATEGORY_CPU, current_time + ) + + # Track code type (Python Code / Native Code) + # This is tri-state: Python (has_gil), Native (on_cpu without gil), or Neither + if has_gil: + self._track_state_transition( + tid, True, self.python_code_start, self.native_code_start, + "Python Code", "Native Code", CATEGORY_CODE_TYPE, current_time + ) + elif on_cpu: + self._track_state_transition( + tid, True, self.native_code_start, self.python_code_start, + "Native Code", "Python Code", CATEGORY_CODE_TYPE, current_time + ) + else: + # Thread is idle (neither has GIL nor on CPU) - close any open code markers + # This handles the third state that _track_state_transition doesn't cover + if tid in self.initialized_threads: + if tid in self.python_code_start: + self._add_marker(tid, "Python Code", self.python_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + if tid in self.native_code_start: + self._add_marker(tid, "Native Code", self.native_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + + # Track "Waiting for GIL" intervals (one-sided tracking) + if gil_requested: + self.gil_wait_start.setdefault(tid, current_time) + elif tid in self.gil_wait_start: + self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), + current_time, CATEGORY_GIL) + + # Track GC events - attribute to all threads that hold the GIL during GC + # (GC is interpreter-wide but runs on whichever thread(s) have the GIL) + # If GIL switches during GC, multiple threads will get GC markers + if gc_collecting and has_gil: + # Start GC marker if not already started for this thread + if tid not in self.gc_start_per_thread: + self.gc_start_per_thread[tid] = current_time + elif tid in self.gc_start_per_thread: + # End GC marker if it was running for this thread + # (either GC finished or thread lost GIL) + self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), + current_time, CATEGORY_GC) + + # Mark thread as initialized after processing all state transitions + self.initialized_threads.add(tid) + + # Categorize: idle if neither has GIL nor on CPU + is_idle = not has_gil and not on_cpu + + # Skip idle threads if skip_idle is enabled + if self.skip_idle and is_idle: + continue + + if not frames: + continue + # Process the stack stack_index = self._process_stack(thread_data, frames) @@ -102,7 +236,6 @@ def collect(self, stack_frames): def _create_thread(self, tid): """Create a new thread structure with processed profile format.""" - import threading # Determine if this is the main thread try: @@ -181,7 +314,7 @@ def _create_thread(self, tid): "functionSize": [], "length": 0, }, - # Markers - processed format + # Markers - processed format (arrays) "markers": { "data": [], "name": [], @@ -215,6 +348,27 @@ def _intern_string(self, s): self.global_string_map[s] = idx return idx + def _add_marker(self, tid, name, start_time, end_time, category): + """Add an interval marker for a specific thread.""" + if tid not in self.threads: + return + + thread_data = self.threads[tid] + duration = end_time - start_time + + name_idx = self._intern_string(name) + markers = thread_data["markers"] + markers["name"].append(name_idx) + markers["startTime"].append(start_time) + markers["endTime"].append(end_time) + markers["phase"].append(1) # 1 = interval marker + markers["category"].append(category) + markers["data"].append({ + "type": name.replace(" ", ""), + "duration": duration, + "tid": tid + }) + def _process_stack(self, thread_data, frames): """Process a stack and return the stack index.""" if not frames: @@ -383,15 +537,63 @@ def _get_or_create_frame(self, thread_data, func_idx, lineno): frame_cache[frame_key] = frame_idx return frame_idx + def _finalize_markers(self): + """Close any open markers at the end of profiling.""" + end_time = self.last_sample_time + + # Close all open markers for each thread using a generic approach + marker_states = [ + (self.has_gil_start, "Has GIL", CATEGORY_GIL), + (self.no_gil_start, "No GIL", CATEGORY_GIL), + (self.on_cpu_start, "On CPU", CATEGORY_CPU), + (self.off_cpu_start, "Off CPU", CATEGORY_CPU), + (self.python_code_start, "Python Code", CATEGORY_CODE_TYPE), + (self.native_code_start, "Native Code", CATEGORY_CODE_TYPE), + (self.gil_wait_start, "Waiting for GIL", CATEGORY_GIL), + (self.gc_start_per_thread, "GC Collecting", CATEGORY_GC), + ] + + for state_dict, marker_name, category in marker_states: + for tid in list(state_dict.keys()): + self._add_marker(tid, marker_name, state_dict[tid], end_time, category) + del state_dict[tid] + def export(self, filename): """Export the profile to a Gecko JSON file.""" + if self.sample_count > 0 and self.last_sample_time > 0: self.interval = self.last_sample_time / self.sample_count - profile = self._build_profile() + # Spinner for progress indication + spinner = itertools.cycle(['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']) + stop_spinner = threading.Event() - with open(filename, "w") as f: - json.dump(profile, f, separators=(",", ":")) + def spin(): + message = 'Building Gecko profile...' + while not stop_spinner.is_set(): + sys.stderr.write(f'\r{next(spinner)} {message}') + sys.stderr.flush() + time.sleep(0.1) + # Clear the spinner line + sys.stderr.write('\r' + ' ' * (len(message) + 3) + '\r') + sys.stderr.flush() + + spinner_thread = threading.Thread(target=spin, daemon=True) + spinner_thread.start() + + try: + # Finalize any open markers before building profile + self._finalize_markers() + + profile = self._build_profile() + + with open(filename, "w") as f: + json.dump(profile, f, separators=(",", ":")) + finally: + stop_spinner.set() + spinner_thread.join(timeout=1.0) + # Small delay to ensure the clear happens + time.sleep(0.01) print(f"Gecko profile written to {filename}") print( @@ -416,6 +618,7 @@ def _build_profile(self): frame_table["length"] = len(frame_table["func"]) func_table["length"] = len(func_table["name"]) resource_table["length"] = len(resource_table["name"]) + thread_data["markers"]["length"] = len(thread_data["markers"]["name"]) # Clean up internal caches del thread_data["_stackCache"] diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 7a0f739a542..5ca68911d8a 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -21,6 +21,7 @@ PROFILING_MODE_WALL = 0 PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks def _parse_mode(mode_string): @@ -136,18 +137,20 @@ def _run_with_sync(original_cmd): class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, all_threads=self.all_threads, mode=mode + self.pid, all_threads=self.all_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, only_active_thread=only_active_threads, mode=mode + self.pid, only_active_thread=only_active_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) @@ -614,14 +617,21 @@ def sample( realtime_stats=False, mode=PROFILING_MODE_WALL, ): + # PROFILING_MODE_ALL implies no skipping at all + if mode == PROFILING_MODE_ALL: + skip_non_matching_threads = False + skip_idle = False + else: + # Determine skip settings based on output format and mode + skip_non_matching_threads = output_format != "gecko" + skip_idle = mode != PROFILING_MODE_WALL + profiler = SampleProfiler( - pid, sample_interval_usec, all_threads=all_threads, mode=mode + pid, sample_interval_usec, all_threads=all_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) profiler.realtime_stats = realtime_stats - # Determine skip_idle for collector compatibility - skip_idle = mode != PROFILING_MODE_WALL - collector = None match output_format: case "pstats": @@ -633,7 +643,8 @@ def sample( collector = FlamegraphCollector(skip_idle=skip_idle) filename = filename or f"flamegraph.{pid}.html" case "gecko": - collector = GeckoCollector(skip_idle=skip_idle) + # Gecko format never skips idle threads to show full thread states + collector = GeckoCollector(skip_idle=False) filename = filename or f"gecko.{pid}.json" case _: raise ValueError(f"Invalid output format: {output_format}") @@ -882,6 +893,10 @@ def main(): if args.format in ("collapsed", "gecko"): _validate_collapsed_format_args(args, parser) + # Validate that --mode is not used with --gecko + if args.format == "gecko" and args.mode != "wall": + parser.error("--mode option is incompatible with --gecko format. Gecko format automatically uses ALL mode (GIL + CPU analysis).") + sort_value = args.sort if args.sort is not None else 2 if args.module is not None and not args.module: @@ -900,7 +915,11 @@ def main(): elif target_count > 1: parser.error("only one target type can be specified: -p/--pid, -m/--module, or script") - mode = _parse_mode(args.mode) + # Use PROFILING_MODE_ALL for gecko format, otherwise parse user's choice + if args.format == "gecko": + mode = PROFILING_MODE_ALL + else: + mode = _parse_mode(args.mode) if args.pid: sample( diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 01720457e61..60e5000cd72 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -23,6 +23,12 @@ PROFILING_MODE_WALL = 0 PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 + +# Thread status flags +THREAD_STATUS_HAS_GIL = (1 << 0) +THREAD_STATUS_ON_CPU = (1 << 1) +THREAD_STATUS_UNKNOWN = (1 << 2) try: from concurrent import interpreters @@ -1763,11 +1769,14 @@ def busy(): for thread_info in interpreter_info.threads: statuses[thread_info.thread_id] = thread_info.status - # Check if sleeper thread is idle and busy thread is running + # Check if sleeper thread is off CPU and busy thread is on CPU + # In the new flags system: + # - sleeper should NOT have ON_CPU flag (off CPU) + # - busy should have ON_CPU flag if (sleeper_tid in statuses and busy_tid in statuses and - statuses[sleeper_tid] == 1 and - statuses[busy_tid] == 0): + not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and + (statuses[busy_tid] & THREAD_STATUS_ON_CPU)): break time.sleep(0.5) # Give a bit of time to let threads settle except PermissionError: @@ -1779,8 +1788,8 @@ def busy(): self.assertIsNotNone(busy_tid, "Busy thread id not received") self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertEqual(statuses[sleeper_tid], 1, "Sleeper thread should be idle (1)") - self.assertEqual(statuses[busy_tid], 0, "Busy thread should be running (0)") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper thread should be off CPU") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy thread should be on CPU") finally: if client_socket is not None: @@ -1875,11 +1884,14 @@ def busy(): for thread_info in interpreter_info.threads: statuses[thread_info.thread_id] = thread_info.status - # Check if sleeper thread is idle (status 2 for GIL mode) and busy thread is running + # Check if sleeper thread doesn't have GIL and busy thread has GIL + # In the new flags system: + # - sleeper should NOT have HAS_GIL flag (waiting for GIL) + # - busy should have HAS_GIL flag if (sleeper_tid in statuses and busy_tid in statuses and - statuses[sleeper_tid] == 2 and - statuses[busy_tid] == 0): + not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and + (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): break time.sleep(0.5) # Give a bit of time to let threads settle except PermissionError: @@ -1891,8 +1903,8 @@ def busy(): self.assertIsNotNone(busy_tid, "Busy thread id not received") self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertEqual(statuses[sleeper_tid], 2, "Sleeper thread should be idle (1)") - self.assertEqual(statuses[busy_tid], 0, "Busy thread should be running (0)") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper thread should not have GIL") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy thread should have GIL") finally: if client_socket is not None: @@ -1900,6 +1912,128 @@ def busy(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") + def test_thread_status_all_mode_detection(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import socket + import threading + import time + import sys + + def sleeper_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"sleeper:" + str(threading.get_native_id()).encode()) + while True: + time.sleep(1) + + def busy_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"busy:" + str(threading.get_native_id()).encode()) + while True: + sum(range(100000)) + + t1 = threading.Thread(target=sleeper_thread) + t2 = threading.Thread(target=busy_thread) + t1.start() + t2.start() + t1.join() + t2.join() + """ + ) + + with os_helper.temp_dir() as tmp_dir: + script_file = make_script(tmp_dir, "script", script) + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.listen(2) + server_socket.settimeout(SHORT_TIMEOUT) + + p = subprocess.Popen( + [sys.executable, script_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + client_sockets = [] + try: + sleeper_tid = None + busy_tid = None + + # Receive thread IDs from the child process + for _ in range(2): + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + line = client_socket.recv(1024) + if line: + if line.startswith(b"sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except Exception: + pass + elif line.startswith(b"busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except Exception: + pass + + server_socket.close() + + attempts = 10 + statuses = {} + try: + unwinder = RemoteUnwinder(p.pid, all_threads=True, mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False) + for _ in range(attempts): + traces = unwinder.get_stack_trace() + # Find threads and their statuses + statuses = {} + for interpreter_info in traces: + for thread_info in interpreter_info.threads: + statuses[thread_info.thread_id] = thread_info.status + + # Check ALL mode provides both GIL and CPU info + # - sleeper should NOT have ON_CPU and NOT have HAS_GIL + # - busy should have ON_CPU and have HAS_GIL + if (sleeper_tid in statuses and + busy_tid in statuses and + not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and + not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and + (statuses[busy_tid] & THREAD_STATUS_ON_CPU) and + (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + self.assertIsNotNone(sleeper_tid, "Sleeper thread id not received") + self.assertIsNotNone(busy_tid, "Busy thread id not received") + self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") + self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") + + # Sleeper thread: off CPU, no GIL + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper should be off CPU") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper should not have GIL") + + # Busy thread: on CPU, has GIL + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy should be on CPU") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy should have GIL") + + finally: + for client_socket in client_sockets: + client_socket.close() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + p.stdout.close() + p.stderr.close() if __name__ == "__main__": diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 0ba6799a1ce..ae9bf3ef2e5 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -63,12 +63,14 @@ def __repr__(self): class MockThreadInfo: """Mock ThreadInfo for testing since the real one isn't accessible.""" - def __init__(self, thread_id, frame_info): + def __init__(self, thread_id, frame_info, status=0, gc_collecting=False): # Default to THREAD_STATE_RUNNING (0) self.thread_id = thread_id self.frame_info = frame_info + self.status = status + self.gc_collecting = gc_collecting def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info})" + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status}, gc_collecting={self.gc_collecting})" class MockInterpreterInfo: @@ -674,6 +676,97 @@ def test_gecko_collector_export(self): self.assertIn("func2", string_array) self.assertIn("other_func", string_array) + def test_gecko_collector_markers(self): + """Test Gecko profile markers for GIL and CPU state tracking.""" + try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_GIL_REQUESTED + except ImportError: + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) + + collector = GeckoCollector() + + # Status combinations for different thread states + HAS_GIL_ON_CPU = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Running Python code + NO_GIL_ON_CPU = THREAD_STATUS_ON_CPU # Running native code + WAITING_FOR_GIL = THREAD_STATUS_GIL_REQUESTED # Waiting for GIL + + # Simulate thread state transitions + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 10, "python_func")], status=HAS_GIL_ON_CPU) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 15, "wait_func")], status=WAITING_FOR_GIL) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 20, "python_func2")], status=HAS_GIL_ON_CPU) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("native.c", 100, "native_func")], status=NO_GIL_ON_CPU) + ]) + ]) + + profile_data = collector._build_profile() + + # Verify we have threads with markers + self.assertIn("threads", profile_data) + self.assertEqual(len(profile_data["threads"]), 1) + thread_data = profile_data["threads"][0] + + # Check markers exist + self.assertIn("markers", thread_data) + markers = thread_data["markers"] + + # Should have marker arrays + self.assertIn("name", markers) + self.assertIn("startTime", markers) + self.assertIn("endTime", markers) + self.assertIn("category", markers) + self.assertGreater(markers["length"], 0, "Should have generated markers") + + # Get marker names from string table + string_array = profile_data["shared"]["stringArray"] + marker_names = [string_array[idx] for idx in markers["name"]] + + # Verify we have different marker types + marker_name_set = set(marker_names) + + # Should have "Has GIL" markers (when thread had GIL) + self.assertIn("Has GIL", marker_name_set, "Should have 'Has GIL' markers") + + # Should have "No GIL" markers (when thread didn't have GIL) + self.assertIn("No GIL", marker_name_set, "Should have 'No GIL' markers") + + # Should have "On CPU" markers (when thread was on CPU) + self.assertIn("On CPU", marker_name_set, "Should have 'On CPU' markers") + + # Should have "Waiting for GIL" markers (when thread was waiting) + self.assertIn("Waiting for GIL", marker_name_set, "Should have 'Waiting for GIL' markers") + + # Verify marker structure + for i in range(markers["length"]): + # All markers should be interval markers (phase = 1) + self.assertEqual(markers["phase"][i], 1, f"Marker {i} should be interval marker") + + # All markers should have valid time range + start_time = markers["startTime"][i] + end_time = markers["endTime"][i] + self.assertLessEqual(start_time, end_time, f"Marker {i} should have valid time range") + + # All markers should have valid category + self.assertGreaterEqual(markers["category"][i], 0, f"Marker {i} should have valid category") + def test_pstats_collector_export(self): collector = PstatsCollector( sample_interval_usec=1000000 @@ -2625,19 +2718,30 @@ def test_mode_validation(self): def test_frames_filtered_with_skip_idle(self): """Test that frames are actually filtered when skip_idle=True.""" + # Import thread status flags + try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU + except ImportError: + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + # Create mock frames with different thread statuses class MockThreadInfoWithStatus: def __init__(self, thread_id, frame_info, status): self.thread_id = thread_id self.frame_info = frame_info self.status = status + self.gc_collecting = False + + # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread + ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU + IDLE_STATUS = 0 # Neither has GIL nor on CPU - # Create test data: running thread, idle thread, and another running thread test_frames = [ MockInterpreterInfo(0, [ - MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], 0), # RUNNING - MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], 1), # IDLE - MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], 0), # RUNNING + MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], ACTIVE_STATUS), + MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], IDLE_STATUS), + MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], ACTIVE_STATUS), ]) ] diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index c6ced39c70c..d190b3c9faf 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -11,6 +11,7 @@ * HEADERS AND INCLUDES * ============================================================================ */ +#include #include #include #include @@ -81,6 +82,8 @@ typedef enum _WIN32_THREADSTATE { #define SIZEOF_TYPE_OBJ sizeof(PyTypeObject) #define SIZEOF_UNICODE_OBJ sizeof(PyUnicodeObject) #define SIZEOF_LONG_OBJ sizeof(PyLongObject) +#define SIZEOF_GC_RUNTIME_STATE sizeof(struct _gc_runtime_state) +#define SIZEOF_INTERPRETER_STATE sizeof(PyInterpreterState) // Calculate the minimum buffer size needed to read interpreter state fields // We need to read code_object_generation and potentially tlbc_generation @@ -178,8 +181,9 @@ static PyStructSequence_Desc CoroInfo_desc = { // ThreadInfo structseq type - replaces 2-tuple (thread_id, frame_info) static PyStructSequence_Field ThreadInfo_fields[] = { {"thread_id", "Thread ID"}, - {"status", "Thread status"}, + {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, {"frame_info", "Frame information"}, + {"gc_collecting", "Whether GC is collecting (interpreter-level)"}, {NULL} }; @@ -187,7 +191,7 @@ static PyStructSequence_Desc ThreadInfo_desc = { "_remote_debugging.ThreadInfo", "Information about a thread", ThreadInfo_fields, - 2 + 3 }; // InterpreterInfo structseq type - replaces 2-tuple (interpreter_id, thread_list) @@ -247,9 +251,16 @@ enum _ThreadState { enum _ProfilingMode { PROFILING_MODE_WALL = 0, PROFILING_MODE_CPU = 1, - PROFILING_MODE_GIL = 2 + PROFILING_MODE_GIL = 2, + PROFILING_MODE_ALL = 3 // Combines GIL + CPU checks }; +// Thread status flags (can be combined) +#define THREAD_STATUS_HAS_GIL (1 << 0) // Thread has the GIL +#define THREAD_STATUS_ON_CPU (1 << 1) // Thread is running on CPU +#define THREAD_STATUS_UNKNOWN (1 << 2) // Status could not be determined +#define THREAD_STATUS_GIL_REQUESTED (1 << 3) // Thread is waiting for the GIL + typedef struct { PyObject_HEAD proc_handle_t handle; @@ -2650,34 +2661,70 @@ unwind_stack_for_thread( long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); - // Calculate thread status based on mode - int status = THREAD_STATE_UNKNOWN; - if (unwinder->mode == PROFILING_MODE_CPU) { - long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); - status = get_thread_status(unwinder, tid, pthread_id); - if (status == -1) { - PyErr_Print(); - PyErr_SetString(PyExc_RuntimeError, "Failed to get thread status"); - goto error; - } - } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Read GC collecting state from the interpreter (before any skip checks) + uintptr_t interp_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.interp); + + // Read the GC runtime state from the interpreter state + uintptr_t gc_addr = interp_addr + unwinder->debug_offsets.interpreter_state.gc; + char gc_state[SIZEOF_GC_RUNTIME_STATE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, gc_addr, unwinder->debug_offsets.gc.size, gc_state) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read GC state"); + goto error; + } + + int gc_collecting = GET_MEMBER(int, gc_state, unwinder->debug_offsets.gc.collecting); + + // Calculate thread status using flags (always) + int status_flags = 0; + + // Check GIL status + int has_gil = 0; + int gil_requested = 0; #ifdef Py_GIL_DISABLED - // All threads are considered running in free threading builds if they have a thread state attached - int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; - status = active ? THREAD_STATE_RUNNING : THREAD_STATE_GIL_WAIT; + int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; + has_gil = active; #else - status = (*current_tstate == gil_holder_tstate) ? THREAD_STATE_RUNNING : THREAD_STATE_GIL_WAIT; + // Read holds_gil directly from thread state + has_gil = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.holds_gil); + + // Check if thread is actively requesting the GIL + if (unwinder->debug_offsets.thread_state.gil_requested != 0) { + gil_requested = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.gil_requested); + } + + // Set GIL_REQUESTED flag if thread is waiting + if (!has_gil && gil_requested) { + status_flags |= THREAD_STATUS_GIL_REQUESTED; + } #endif - } else { - // PROFILING_MODE_WALL - all threads are considered running - status = THREAD_STATE_RUNNING; + if (has_gil) { + status_flags |= THREAD_STATUS_HAS_GIL; + } + + // Assert that we never have both HAS_GIL and GIL_REQUESTED set at the same time + // This would indicate a race condition in the GIL state tracking + assert(!(has_gil && gil_requested)); + + // Check CPU status + long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); + int cpu_status = get_thread_status(unwinder, tid, pthread_id); + if (cpu_status == -1) { + status_flags |= THREAD_STATUS_UNKNOWN; + } else if (cpu_status == THREAD_STATE_RUNNING) { + status_flags |= THREAD_STATUS_ON_CPU; } // Check if we should skip this thread based on mode int should_skip = 0; - if (unwinder->skip_non_matching_threads && status != THREAD_STATE_RUNNING && - (unwinder->mode == PROFILING_MODE_CPU || unwinder->mode == PROFILING_MODE_GIL)) { - should_skip = 1; + if (unwinder->skip_non_matching_threads) { + if (unwinder->mode == PROFILING_MODE_CPU) { + // Skip if not on CPU + should_skip = !(status_flags & THREAD_STATUS_ON_CPU); + } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Skip if doesn't have GIL + should_skip = !(status_flags & THREAD_STATUS_HAS_GIL); + } + // PROFILING_MODE_WALL and PROFILING_MODE_ALL never skip } if (should_skip) { @@ -2719,16 +2766,25 @@ unwind_stack_for_thread( goto error; } - PyObject *py_status = PyLong_FromLong(status); + // Always use status_flags + PyObject *py_status = PyLong_FromLong(status_flags); if (py_status == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread status"); goto error; } - PyErr_Print(); + PyObject *py_gc_collecting = PyBool_FromLong(gc_collecting); + if (py_gc_collecting == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create gc_collecting"); + Py_DECREF(py_status); + goto error; + } + + // py_status contains status flags (bitfield) PyStructSequence_SetItem(result, 0, thread_id); PyStructSequence_SetItem(result, 1, py_status); // Steals reference PyStructSequence_SetItem(result, 2, frame_info); // Steals reference + PyStructSequence_SetItem(result, 3, py_gc_collecting); // Steals reference cleanup_stack_chunks(&chunks); return result; @@ -3401,6 +3457,21 @@ _remote_debugging_exec(PyObject *m) if (rc < 0) { return -1; } + + // Add thread status flag constants + if (PyModule_AddIntConstant(m, "THREAD_STATUS_HAS_GIL", THREAD_STATUS_HAS_GIL) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_ON_CPU", THREAD_STATUS_ON_CPU) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_UNKNOWN", THREAD_STATUS_UNKNOWN) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_GIL_REQUESTED", THREAD_STATUS_GIL_REQUESTED) < 0) { + return -1; + } + if (RemoteDebugging_InitState(st) < 0) { return -1; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 9b6506ac332..f6ada3892f8 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -207,6 +207,7 @@ drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) _Py_atomic_store_int_relaxed(&gil->locked, 0); if (tstate != NULL) { tstate->holds_gil = 0; + tstate->gil_requested = 0; } COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); @@ -320,6 +321,8 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); + tstate->gil_requested = 1; + int drop_requested = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -407,6 +410,7 @@ take_gil(PyThreadState *tstate) } assert(_PyThreadState_CheckConsistency(tstate)); + tstate->gil_requested = 0; tstate->holds_gil = 1; _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); update_eval_breaker_for_thread(interp, tstate); From 336366fd7ca61858572fdb78e2bd79014b215f19 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 17 Nov 2025 05:39:00 -0800 Subject: [PATCH 217/417] GH-140643: Add `` and `` frames to the sampling profiler (#141108) - Introduce a new field in the GC state to store the frame that initiated garbage collection. - Update RemoteUnwinder to include options for including "" and "" frames in the stack trace. - Modify the sampling profiler to accept parameters for controlling the inclusion of native and GC frames. - Enhance the stack collector to properly format and append these frames during profiling. - Add tests to verify the correct behavior of the profiler with respect to native and GC frames, including options to exclude them. Co-authored-by: Pablo Galindo Salgado --- Doc/library/profile.rst | 12 +- Include/internal/pycore_debug_offsets.h | 2 + .../pycore_global_objects_fini_generated.h | 4 + Include/internal/pycore_global_strings.h | 4 + Include/internal/pycore_interp_structs.h | 3 + Include/internal/pycore_interpframe_structs.h | 1 - .../internal/pycore_runtime_init_generated.h | 4 + .../internal/pycore_unicodeobject_generated.h | 16 ++ Lib/profiling/sampling/flamegraph.js | 28 ++- Lib/profiling/sampling/sample.py | 26 ++- Lib/profiling/sampling/stack_collector.py | 18 +- Lib/test/test_external_inspection.py | 2 + .../test_profiling/test_sampling_profiler.py | 208 +++++++++++++++++- ...-11-05-19-50-37.gh-issue-140643.QCEOqG.rst | 3 + Modules/_remote_debugging_module.c | 170 +++++++++----- Modules/clinic/_remote_debugging_module.c.h | 46 +++- Python/gc.c | 2 + Python/gc_free_threading.c | 2 + 18 files changed, 465 insertions(+), 86 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index faf8079db3d..5bf36b13c6d 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -265,6 +265,14 @@ Profile with real-time sampling statistics:: Sample all threads in the process instead of just the main thread +.. option:: --native + + Include artificial ```` frames to denote calls to non-Python code. + +.. option:: --no-gc + + Don't include artificial ```` frames to denote active garbage collection. + .. option:: --realtime-stats Print real-time sampling statistics during profiling @@ -349,7 +357,7 @@ This section documents the programmatic interface for the :mod:`!profiling.sampl For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information about statistical profiling, see :ref:`statistical-profiling` -.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) +.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False, native=False, gc=True) Sample a Python process and generate profiling data. @@ -367,6 +375,8 @@ about statistical profiling, see :ref:`statistical-profiling` :param bool show_summary: Whether to show summary statistics (default: True) :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') :param bool realtime_stats: Whether to display real-time statistics (default: False) + :param bool native: Whether to include ```` frames (default: False) + :param bool gc: Whether to include ```` frames (default: True) :raises ValueError: If output_format is not 'pstats' or 'collapsed' diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index f6d50bf5df7..0f17bf17f82 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -212,6 +212,7 @@ typedef struct _Py_DebugOffsets { struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; } gc; // Generator object offset; @@ -355,6 +356,7 @@ typedef struct _Py_DebugOffsets { .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 92ded14891a..ecef4364cc3 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1326,10 +1326,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); @@ -1763,6 +1765,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); @@ -1906,6 +1909,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cd21b0847b7..4dd73291df4 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,10 +46,12 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") @@ -486,6 +488,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) @@ -629,6 +632,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9e4504479cd..f861d3abd96 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -212,6 +212,9 @@ struct _gc_runtime_state { struct gc_generation_stats generation_stats[NUM_GENERATIONS]; /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194..38510685f40 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 50d82d0a365..08f8d0e59d1 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1321,10 +1321,12 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ @@ -1761,6 +1763,7 @@ extern "C" { INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ @@ -1904,6 +1907,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b4d920154b6..b1e57126b92 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1732,6 +1732,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2304,6 +2308,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3236,6 +3244,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3260,6 +3272,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 95ad7ca6184..670ca22d442 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -151,17 +151,22 @@ function createPythonTooltip(data) { const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); const filename = resolveString(d.data.filename) || ""; + // Don't show file location for special frames like and + const isSpecialFrame = filename === "~"; + const fileLocationHTML = isSpecialFrame ? "" : ` +

+ ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} +
`; + const tooltipHTML = `
${funcname}
-
- ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} -
+ ${fileLocationHTML}
Execution Time: @@ -474,14 +479,23 @@ function populateStats(data) { if (i < hotSpots.length && hotSpots[i]) { const hotspot = hotSpots[i]; const filename = hotspot.filename || 'unknown'; - const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; const lineno = hotspot.lineno ?? '?'; let funcDisplay = hotspot.funcname || 'unknown'; if (funcDisplay.length > 35) { funcDisplay = funcDisplay.substring(0, 32) + '...'; } - document.getElementById(`hotspot-file-${num}`).textContent = `${basename}:${lineno}`; + // Don't show file:line for special frames like and + const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?'); + let fileDisplay; + if (isSpecialFrame) { + fileDisplay = '--'; + } else { + const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; + fileDisplay = `${basename}:${lineno}`; + } + + document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay; document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay; document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`; } else { diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5ca68911d8a..713931a639d 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -137,19 +137,19 @@ def _run_with_sync(original_cmd): class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, all_threads=self.all_threads, mode=mode, + self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, only_active_thread=only_active_threads, mode=mode, + self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) # Track sample intervals and total sample count @@ -616,6 +616,8 @@ def sample( output_format="pstats", realtime_stats=False, mode=PROFILING_MODE_WALL, + native=False, + gc=True, ): # PROFILING_MODE_ALL implies no skipping at all if mode == PROFILING_MODE_ALL: @@ -627,7 +629,7 @@ def sample( skip_idle = mode != PROFILING_MODE_WALL profiler = SampleProfiler( - pid, sample_interval_usec, all_threads=all_threads, mode=mode, + pid, sample_interval_usec, all_threads=all_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) profiler.realtime_stats = realtime_stats @@ -717,6 +719,8 @@ def wait_for_process_and_sample(pid, sort_value, args): output_format=args.format, realtime_stats=args.realtime_stats, mode=mode, + native=args.native, + gc=args.gc, ) @@ -767,9 +771,19 @@ def main(): sampling_group.add_argument( "--realtime-stats", action="store_true", - default=False, help="Print real-time sampling statistics (Hz, mean, min, max, stdev) during profiling", ) + sampling_group.add_argument( + "--native", + action="store_true", + help="Include artificial \"\" frames to denote calls to non-Python code.", + ) + sampling_group.add_argument( + "--no-gc", + action="store_false", + dest="gc", + help="Don't include artificial \"\" frames to denote active garbage collection.", + ) # Mode options mode_group = parser.add_argument_group("Mode options") @@ -934,6 +948,8 @@ def main(): output_format=args.format, realtime_stats=args.realtime_stats, mode=mode, + native=args.native, + gc=args.gc, ) elif args.module or args.args: if args.module: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index bc38151e067..1436811976a 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -36,10 +36,16 @@ def process_frames(self, frames, thread_id): def export(self, filename): lines = [] for (call_tree, thread_id), count in self.stack_counter.items(): - stack_str = ";".join( - f"{os.path.basename(f[0])}:{f[2]}:{f[1]}" for f in call_tree - ) - lines.append((f"tid:{thread_id};{stack_str}", count)) + parts = [f"tid:{thread_id}"] + for file, line, func in call_tree: + # This is what pstats does for "special" frames: + if file == "~" and line == 0: + part = func + else: + part = f"{os.path.basename(file)}:{func}:{line}" + parts.append(part) + stack_str = ";".join(parts) + lines.append((stack_str, count)) lines.sort(key=lambda x: (-x[1], x[0])) @@ -98,6 +104,10 @@ def export(self, filename): def _format_function_name(func): filename, lineno, funcname = func + # Special frames like and should not show file:line + if filename == "~" and lineno == 0: + return funcname + if len(filename) > 50: parts = filename.split("/") if len(parts) > 2: diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 60e5000cd72..7decd8f32d5 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -159,6 +159,8 @@ def foo(): FrameInfo([script_name, 12, "baz"]), FrameInfo([script_name, 9, "bar"]), FrameInfo([threading.__file__, ANY, "Thread.run"]), + FrameInfo([threading.__file__, ANY, "Thread._bootstrap_inner"]), + FrameInfo([threading.__file__, ANY, "Thread._bootstrap"]), ] # Is possible that there are more threads, so we check that the # expected stack traces are in the result (looking at you Windows!) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index ae9bf3ef2e5..a24dbb55cd7 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -2025,7 +2025,6 @@ def test_sample_target_script(self): # Should see some of our test functions self.assertIn("slow_fibonacci", output) - def test_sample_target_module(self): tempdir = tempfile.TemporaryDirectory(delete=False) self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) @@ -2264,7 +2263,9 @@ def test_cli_module_argument_parsing(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2292,7 +2293,9 @@ def test_cli_module_with_arguments(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2320,7 +2323,9 @@ def test_cli_script_argument_parsing(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2420,7 +2425,9 @@ def test_cli_module_with_profiler_options(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2454,7 +2461,9 @@ def test_cli_script_with_profiler_options(self): show_summary=True, output_format="collapsed", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) def test_cli_empty_module_name(self): @@ -2666,7 +2675,9 @@ def test_argument_parsing_basic(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) def test_sort_options(self): @@ -3121,6 +3132,187 @@ def test_parse_mode_function(self): @requires_subprocess() @skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + """Class that triggers GC with expensive finalizer (callback).""" + def __init__(self): + self.cycle = self + + def __del__(self): + # CPU-intensive work in the finalizer callback + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +def main_loop(): + """Main loop that triggers GC with expensive callback.""" + while True: + ExpensiveGarbage() + gc.collect() + +if __name__ == "__main__": + main_loop() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = ''' +import operator + +def main_loop(): + while True: + # Native code in the middle of the stack: + operator.call(inner) + +def inner(): + # Python code at the top of the stack: + for _ in range(1_000_0000): + pass + +if __name__ == "__main__": + main_loop() +''' + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.native_test_script) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=1000, + native=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("", output) + + class TestProcessPoolExecutorSupport(unittest.TestCase): """ Test that ProcessPoolExecutor works correctly with profiling.sampling. @@ -3161,7 +3353,5 @@ def worker(x): self.assertIn("Results: [2, 4, 6]", stdout) self.assertNotIn("Can't pickle", stderr) - - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst new file mode 100644 index 00000000000..e1202dd1a17 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst @@ -0,0 +1,3 @@ +Add support for ```` and ```` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index d190b3c9faf..51b3c6bac02 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -26,8 +26,9 @@ #include "Python.h" #include // _Py_DebugOffsets #include // FRAME_SUSPENDED_YIELD_FROM -#include // FRAME_OWNED_BY_CSTACK +#include // FRAME_OWNED_BY_INTERPRETER #include // struct llist_node +#include // _PyLong_GetZero #include // Py_TAG_BITS #include "../Python/remote_debug.h" @@ -92,14 +93,16 @@ typedef enum _WIN32_THREADSTATE { #endif #ifdef Py_GIL_DISABLED -#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ - offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) #else -#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ - offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) #endif #define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) @@ -276,6 +279,8 @@ typedef struct { int only_active_thread; int mode; // Use enum _ProfilingMode values int skip_non_matching_threads; // New option to skip threads that don't match mode + int native; + int gc; RemoteDebuggingState *cached_state; // Cached module state #ifdef Py_GIL_DISABLED // TLBC cache invalidation tracking @@ -1812,6 +1817,25 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L * CODE OBJECT AND FRAME PARSING FUNCTIONS * ============================================================================ */ +static PyObject * +make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *line, + PyObject *func) +{ + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *info = PyStructSequence_New(state->FrameInfo_Type); + if (info == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo"); + return NULL; + } + Py_INCREF(file); + Py_INCREF(line); + Py_INCREF(func); + PyStructSequence_SetItem(info, 0, file); + PyStructSequence_SetItem(info, 1, line); + PyStructSequence_SetItem(info, 2, func); + return info; +} + static int parse_code_object(RemoteUnwinderObject *unwinder, PyObject **result, @@ -1825,8 +1849,6 @@ parse_code_object(RemoteUnwinderObject *unwinder, PyObject *func = NULL; PyObject *file = NULL; PyObject *linetable = NULL; - PyObject *lineno = NULL; - PyObject *tuple = NULL; #ifdef Py_GIL_DISABLED // In free threading builds, code object addresses might have the low bit set @@ -1948,25 +1970,18 @@ parse_code_object(RemoteUnwinderObject *unwinder, info.lineno = -1; } - lineno = PyLong_FromLong(info.lineno); + PyObject *lineno = PyLong_FromLong(info.lineno); if (!lineno) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create line number object"); goto error; } - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - tuple = PyStructSequence_New(state->FrameInfo_Type); + PyObject *tuple = make_frame_info(unwinder, meta->file_name, lineno, meta->func_name); + Py_DECREF(lineno); if (!tuple) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo for code object"); goto error; } - Py_INCREF(meta->func_name); - Py_INCREF(meta->file_name); - PyStructSequence_SetItem(tuple, 0, meta->file_name); - PyStructSequence_SetItem(tuple, 1, lineno); - PyStructSequence_SetItem(tuple, 2, meta->func_name); - *result = tuple; return 0; @@ -1974,8 +1989,6 @@ parse_code_object(RemoteUnwinderObject *unwinder, Py_XDECREF(func); Py_XDECREF(file); Py_XDECREF(linetable); - Py_XDECREF(lineno); - Py_XDECREF(tuple); return -1; } @@ -2116,6 +2129,7 @@ parse_frame_from_chunks( PyObject **result, uintptr_t address, uintptr_t *previous_frame, + uintptr_t *stackpointer, StackChunkList *chunks ) { void *frame_ptr = find_frame_in_chunks(chunks, address); @@ -2126,6 +2140,7 @@ parse_frame_from_chunks( char *frame = (char *)frame_ptr; *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + *stackpointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.stackpointer); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame_ptr, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); if (frame_valid != 1) { @@ -2238,8 +2253,7 @@ is_frame_valid( void* frame = (void*)frame_addr; - if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_CSTACK || - GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { return 0; // C frame } @@ -2458,8 +2472,9 @@ process_frame_chain( RemoteUnwinderObject *unwinder, uintptr_t initial_frame_addr, StackChunkList *chunks, - PyObject *frame_info -) { + PyObject *frame_info, + uintptr_t gc_frame) +{ uintptr_t frame_addr = initial_frame_addr; uintptr_t prev_frame_addr = 0; const size_t MAX_FRAMES = 1024; @@ -2468,6 +2483,7 @@ process_frame_chain( while ((void*)frame_addr != NULL) { PyObject *frame = NULL; uintptr_t next_frame_addr = 0; + uintptr_t stackpointer = 0; if (++frame_count > MAX_FRAMES) { PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); @@ -2476,7 +2492,7 @@ process_frame_chain( } // Try chunks first, fallback to direct memory read - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, chunks) < 0) { PyErr_Clear(); uintptr_t address_of_code_object = 0; if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { @@ -2484,26 +2500,63 @@ process_frame_chain( return -1; } } - - if (!frame) { - break; - } - - if (prev_frame_addr && frame_addr != prev_frame_addr) { - PyErr_Format(PyExc_RuntimeError, - "Broken frame chain: expected frame at 0x%lx, got 0x%lx", - prev_frame_addr, frame_addr); - Py_DECREF(frame); - set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain consistency check failed"); + if (frame == NULL && PyList_GET_SIZE(frame_info) == 0) { + // If the first frame is missing, the chain is broken: + const char *e = "Failed to parse initial frame in chain"; + PyErr_SetString(PyExc_RuntimeError, e); return -1; } - - if (PyList_Append(frame_info, frame) == -1) { - Py_DECREF(frame); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to frame info list"); - return -1; + PyObject *extra_frame = NULL; + // This frame kicked off the current GC collection: + if (unwinder->gc && frame_addr == gc_frame) { + _Py_DECLARE_STR(gc, ""); + extra_frame = &_Py_STR(gc); + } + // Otherwise, check for native frames to insert: + else if (unwinder->native && + // We've reached an interpreter trampoline frame: + frame == NULL && + // Bottommost frame is always native, so skip that one: + next_frame_addr && + // Only suppress native frames if GC tracking is enabled and the next frame will be a GC frame: + !(unwinder->gc && next_frame_addr == gc_frame)) + { + _Py_DECLARE_STR(native, ""); + extra_frame = &_Py_STR(native); + } + if (extra_frame) { + // Use "~" as file and 0 as line, since that's what pstats uses: + PyObject *extra_frame_info = make_frame_info( + unwinder, _Py_LATIN1_CHR('~'), _PyLong_GetZero(), extra_frame); + if (extra_frame_info == NULL) { + return -1; + } + int error = PyList_Append(frame_info, extra_frame_info); + Py_DECREF(extra_frame_info); + if (error) { + const char *e = "Failed to append extra frame to frame info list"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + } + if (frame) { + if (prev_frame_addr && frame_addr != prev_frame_addr) { + const char *f = "Broken frame chain: expected frame at 0x%lx, got 0x%lx"; + PyErr_Format(PyExc_RuntimeError, f, prev_frame_addr, frame_addr); + Py_DECREF(frame); + const char *e = "Frame chain consistency check failed"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + + if (PyList_Append(frame_info, frame) == -1) { + Py_DECREF(frame); + const char *e = "Failed to append frame to frame info list"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + Py_DECREF(frame); } - Py_DECREF(frame); prev_frame_addr = next_frame_addr; frame_addr = next_frame_addr; @@ -2644,7 +2697,8 @@ static PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, - uintptr_t gil_holder_tstate + uintptr_t gil_holder_tstate, + uintptr_t gc_frame ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; @@ -2746,7 +2800,7 @@ unwind_stack_for_thread( goto error; } - if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info) < 0) { + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, gc_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); goto error; } @@ -2818,6 +2872,8 @@ _remote_debugging.RemoteUnwinder.__init__ mode: int = 0 debug: bool = False skip_non_matching_threads: bool = True + native: bool = False + gc: bool = False Initialize a new RemoteUnwinder object for debugging a remote Python process. @@ -2832,6 +2888,10 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. lead to the exception. skip_non_matching_threads: If True, skip threads that don't match the selected mode. If False, include all threads regardless of mode. + native: If True, include artificial "" frames to denote calls to + non-Python code. + gc: If True, include artificial "" frames to denote active garbage + collection. The RemoteUnwinder provides functionality to inspect and debug a running Python process, including examining thread states, stack frames and other runtime data. @@ -2848,8 +2908,9 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, int only_active_thread, int mode, int debug, - int skip_non_matching_threads) -/*[clinic end generated code: output=abf5ea5cd58bcb36 input=08fb6ace023ec3b5]*/ + int skip_non_matching_threads, + int native, int gc) +/*[clinic end generated code: output=e9eb6b4df119f6e0 input=606d099059207df2]*/ { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { @@ -2866,6 +2927,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, } #endif + self->native = native; + self->gc = gc; self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -3026,6 +3089,13 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self goto exit; } + uintptr_t gc_frame = 0; + if (self->gc) { + gc_frame = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.gc + + self->debug_offsets.gc.frame); + } + int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); @@ -3085,7 +3155,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } while (current_tstate != 0) { - PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate); + PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, + gil_holder_tstate, + gc_frame); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL) && !PyErr_Occurred()) { diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h index 7dd54e31248..60adb357e32 100644 --- a/Modules/clinic/_remote_debugging_module.c.h +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -11,7 +11,8 @@ preserve PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" -" mode=0, debug=False, skip_non_matching_threads=True)\n" +" mode=0, debug=False, skip_non_matching_threads=True,\n" +" native=False, gc=False)\n" "--\n" "\n" "Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" @@ -27,6 +28,10 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, " lead to the exception.\n" " skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" " If False, include all threads regardless of mode.\n" +" native: If True, include artificial \"\" frames to denote calls to\n" +" non-Python code.\n" +" gc: If True, include artificial \"\" frames to denote active garbage\n" +" collection.\n" "\n" "The RemoteUnwinder provides functionality to inspect and debug a running Python\n" "process, including examining thread states, stack frames and other runtime data.\n" @@ -42,7 +47,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, int only_active_thread, int mode, int debug, - int skip_non_matching_threads); + int skip_non_matching_threads, + int native, int gc); static int _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -50,7 +56,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 6 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -59,7 +65,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), }, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -68,14 +74,14 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", NULL}; + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "RemoteUnwinder", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[8]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -85,6 +91,8 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int mode = 0; int debug = 0; int skip_non_matching_threads = 1; + int native = 0; + int gc = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -134,12 +142,30 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje goto skip_optional_kwonly; } } - skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); - if (skip_non_matching_threads < 0) { + if (fastargs[5]) { + skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); + if (skip_non_matching_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + native = PyObject_IsTrue(fastargs[6]); + if (native < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + gc = PyObject_IsTrue(fastargs[7]); + if (gc < 0) { goto exit; } skip_optional_kwonly: - return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads); + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc); exit: return return_value; @@ -321,4 +347,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=2caefeddf7683d32 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=99fed5c94cf36881 input=a9049054013a1b77]*/ diff --git a/Python/gc.c b/Python/gc.c index 03a5d7366ea..064f9406e0a 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2074,6 +2074,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) // Don't start a garbage collection if one is already in progress. return 0; } + gcstate->frame = tstate->current_frame; struct gc_collection_stats stats = { 0 }; if (reason != _Py_GC_REASON_SHUTDOWN) { @@ -2119,6 +2120,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) } #endif validate_spaces(gcstate); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); if (gcstate->debug & _PyGC_DEBUG_STATS) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b183062eff7..7724676c242 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2359,6 +2359,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_atomic_store_int(&gcstate->collecting, 0); return 0; } + gcstate->frame = tstate->current_frame; assert(generation >= 0 && generation < NUM_GENERATIONS); @@ -2447,6 +2448,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } assert(!_PyErr_Occurred(tstate)); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); return n + m; } From f6dd9c12a8ba391cbbcc793411ac7dcfa6e01028 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 17 Nov 2025 05:41:22 -0800 Subject: [PATCH 218/417] GH-139914: Handle stack growth direction on HPPA (GH-140028) Adapted from a patch for Python 3.14 submitted to the Debian BTS by John https://bugs.debian.org/1105111#20 Co-authored-by: John David Anglin --- Include/internal/pycore_ceval.h | 8 ++++ Include/internal/pycore_pystate.h | 4 ++ Include/pyport.h | 6 +++ Lib/test/test_call.py | 9 +++- ...-10-13-13-54-19.gh-issue-139914.M-y_3E.rst | 1 + Modules/_testcapimodule.c | 4 ++ Python/ceval.c | 43 +++++++++++++++++-- configure | 13 ++++++ configure.ac | 8 ++++ pyconfig.h.in | 3 ++ 10 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 33b9fd053f7..47c42fccdc2 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -217,7 +217,11 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN return here_addr < _tstate->c_stack_soft_limit; +#else + return here_addr > _tstate->c_stack_soft_limit; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -249,7 +253,11 @@ static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit; +#else + return here_addr >= _tstate->c_stack_soft_limit; +#endif } static inline void _Py_LeaveRecursiveCall(void) { diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index cab458f8402..189a8dde9f0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -331,7 +331,11 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); +#if _Py_STACK_GROWS_DOWN return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } #ifdef __cplusplus diff --git a/Include/pyport.h b/Include/pyport.h index e77b39026a5..b250f9e308f 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -677,4 +677,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 31e58e825be..f42526aee19 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1048,9 +1048,14 @@ def get_sp(): this_sp = _testinternalcapi.get_stack_pointer() lower_sp = _testcapi.pyobject_vectorcall(get_sp, (), ()) - self.assertLess(lower_sp, this_sp) + if _testcapi._Py_STACK_GROWS_DOWN: + self.assertLess(lower_sp, this_sp) + safe_margin = this_sp - lower_sp + else: + self.assertGreater(lower_sp, this_sp) + safe_margin = lower_sp - this_sp # Add an (arbitrary) extra 25% for safety - safe_margin = (this_sp - lower_sp) * 5 / 4 + safe_margin = safe_margin * 5 / 4 self.assertLess(safe_margin, _testinternalcapi.get_stack_margin()) @skip_on_s390x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst new file mode 100644 index 00000000000..7529108d5d4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst @@ -0,0 +1 @@ +Restore support for HP PA-RISC, which has an upwards-growing stack. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 22cd731d410..c14f925b4e7 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3359,6 +3359,10 @@ _testcapi_exec(PyObject *m) PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + if (PyModule_AddIntMacro(m, _Py_STACK_GROWS_DOWN)) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_single_input)) { return -1; } diff --git a/Python/ceval.c b/Python/ceval.c index 31b81a37464..25294ebd993 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -351,13 +351,21 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr > _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES) { +#else + if (here_addr <= _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES) { +#endif return 0; } if (_tstate->c_stack_hard_limit == 0) { _Py_InitializeRecursionLimits(tstate); } +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES; +#endif } void @@ -365,7 +373,11 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { +#else + if (here_addr > _tstate->c_stack_hard_limit) { +#endif Py_FatalError("Unchecked stack overflow."); } } @@ -496,18 +508,33 @@ tstate_set_stack(PyThreadState *tstate, #ifdef _Py_THREAD_SANITIZER // Thread sanitizer crashes if we use more than half the stack. uintptr_t stacksize = top - base; - base += stacksize / 2; +# if _Py_STACK_GROWS_DOWN + base += stacksize/2; +# else + top -= stacksize/2; +# endif #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN _tstate->c_stack_top = top; _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; - -#ifndef NDEBUG +# ifndef NDEBUG // Sanity checks _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); assert(ts->c_stack_soft_limit < ts->c_stack_top); +# endif +#else + _tstate->c_stack_top = base; + _tstate->c_stack_hard_limit = top - _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = top - _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit >= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit > ts->c_stack_top); +# endif #endif } @@ -568,9 +595,15 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) uintptr_t here_addr = _Py_get_machine_stack_pointer(); assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + if (here_addr > _tstate->c_stack_hard_limit) { + /* Overflowing while handling an overflow. Give up. */ + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -579,7 +612,11 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return 0; } else { +#if _Py_STACK_GROWS_DOWN int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "Stack overflow (used %d kB)%s", diff --git a/configure b/configure index eeb24c1d844..a4514f80c3a 100755 --- a/configure +++ b/configure @@ -967,6 +967,7 @@ LDLIBRARY LIBRARY BUILDEXEEXT NO_AS_NEEDED +_Py_STACK_GROWS_DOWN MULTIARCH_CPPFLAGS PLATFORM_TRIPLET MULTIARCH @@ -7213,6 +7214,18 @@ if test x$MULTIARCH != x; then fi +# Guess C stack direction +case $host in #( + hppa*) : + _Py_STACK_GROWS_DOWN=0 ;; #( + *) : + _Py_STACK_GROWS_DOWN=1 ;; +esac + +printf "%s\n" "#define _Py_STACK_GROWS_DOWN $_Py_STACK_GROWS_DOWN" >>confdefs.h + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PEP 11 support tier" >&5 printf %s "checking for PEP 11 support tier... " >&6; } case $host/$ac_cv_cc_name in #( diff --git a/configure.ac b/configure.ac index 92adc44da0d..a059a07bec2 100644 --- a/configure.ac +++ b/configure.ac @@ -1202,6 +1202,14 @@ if test x$MULTIARCH != x; then fi AC_SUBST([MULTIARCH_CPPFLAGS]) +# Guess C stack direction +AS_CASE([$host], + [hppa*], [_Py_STACK_GROWS_DOWN=0], + [_Py_STACK_GROWS_DOWN=1]) +AC_DEFINE_UNQUOTED([_Py_STACK_GROWS_DOWN], [$_Py_STACK_GROWS_DOWN], + [Define to 1 if the machine stack grows down (default); 0 if it grows up.]) +AC_SUBST([_Py_STACK_GROWS_DOWN]) + dnl Support tiers according to https://peps.python.org/pep-0011/ dnl dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. diff --git a/pyconfig.h.in b/pyconfig.h.in index fb12079bafa..8a9f5ca8ec8 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -2050,6 +2050,9 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ +#undef _Py_STACK_GROWS_DOWN + /* Define if you want to use tail-calling interpreters in CPython. */ #undef _Py_TAIL_CALL_INTERP From 3d148059479b28a21f8eae6abf6d1bcc91ab8cbb Mon Sep 17 00:00:00 2001 From: "R.C.M" Date: Mon, 17 Nov 2025 09:42:26 -0500 Subject: [PATCH 219/417] gh-130693: Support more options for search in tkinter.Text (GH-130848) * Add parameters nolinestop and strictlimits in the tkinter.Text.search() method. * Add the tkinter.Text.search_all() method. * Add more tests for tkinter.Text.search(). * stopindex is now only ignored if it is None. --- Doc/whatsnew/3.15.rst | 13 ++ Lib/test/test_tkinter/test_text.py | 114 +++++++++++++++++- Lib/tkinter/__init__.py | 34 +++++- ...-03-04-17-19-26.gh-issue-130693.Kv01r8.rst | 1 + 4 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9393b65ed8e..cf5bef15203 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -734,6 +734,19 @@ timeit :ref:`environment variables `. (Contributed by Yi Hong in :gh:`139374`.) +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`) types ------ diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index b26956930d3..d579cca95ee 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -34,12 +34,116 @@ def test_search(self): # Invalid text index. self.assertRaises(tkinter.TclError, text.search, '', 0) + self.assertRaises(tkinter.TclError, text.search, '', '') + self.assertRaises(tkinter.TclError, text.search, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 'invalid') - # Check if we are getting the indices as strings -- you are likely - # to get Tcl_Obj under Tk 8.5 if Tkinter doesn't convert it. - text.insert('1.0', 'hi-test') - self.assertEqual(text.search('-test', '1.0', 'end'), '1.2') - self.assertEqual(text.search('test', '1.0', 'end'), '1.3') + text.insert('1.0', + 'This is a test. This is only a test.\n' + 'Another line.\n' + 'Yet another line.\n' + '64-bit') + + self.assertEqual(text.search('test', '1.0'), '1.10') + self.assertEqual(text.search('test', '1.0', 'end'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.10'), '') + self.assertEqual(text.search('test', '1.11'), '1.31') + self.assertEqual(text.search('test', '1.32', 'end'), '') + self.assertEqual(text.search('test', '1.32'), '1.10') + + self.assertEqual(text.search('', '1.0'), '1.0') # empty pattern + self.assertEqual(text.search('nonexistent', '1.0'), '') + self.assertEqual(text.search('-bit', '1.0'), '4.2') # starts with a hyphen + + self.assertEqual(text.search('line', '3.0'), '3.12') + self.assertEqual(text.search('line', '3.0', forwards=True), '3.12') + self.assertEqual(text.search('line', '3.0', backwards=True), '2.8') + self.assertEqual(text.search('line', '3.0', forwards=True, backwards=True), '2.8') + + self.assertEqual(text.search('t.', '1.0'), '1.13') + self.assertEqual(text.search('t.', '1.0', exact=True), '1.13') + self.assertEqual(text.search('t.', '1.0', regexp=True), '1.10') + self.assertEqual(text.search('t.', '1.0', exact=True, regexp=True), '1.10') + + self.assertEqual(text.search('TEST', '1.0'), '') + self.assertEqual(text.search('TEST', '1.0', nocase=True), '1.10') + + self.assertEqual(text.search('.*line', '1.0', regexp=True), '2.0') + self.assertEqual(text.search('.*line', '1.0', regexp=True, nolinestop=True), '1.0') + + self.assertEqual(text.search('test', '1.0', '1.13'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.13', strictlimits=True), '') + self.assertEqual(text.search('test', '1.0', '1.14', strictlimits=True), '1.10') + + var = tkinter.Variable(self.root) + self.assertEqual(text.search('test', '1.0', count=var), '1.10') + self.assertEqual(var.get(), 4 if self.wantobjects else '4') + + # TODO: Add test for elide=True + + def test_search_all(self): + text = self.text + + # pattern and index are obligatory arguments. + self.assertRaises(tkinter.TclError, text.search_all, None, '1.0') + self.assertRaises(tkinter.TclError, text.search_all, 'a', None) + self.assertRaises(tkinter.TclError, text.search_all, None, None) + + # Keyword-only arguments + self.assertRaises(TypeError, text.search_all, 'a', '1.0', 'end', None) + + # Invalid text index. + self.assertRaises(tkinter.TclError, text.search_all, '', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '') + self.assertRaises(tkinter.TclError, text.search_all, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 'invalid') + + def eq(res, expected): + self.assertIsInstance(res, tuple) + self.assertEqual([str(i) for i in res], expected) + + text.insert('1.0', 'ababa\naba\n64-bit') + + eq(text.search_all('aba', '1.0'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', 'end'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.1', 'end'), ['1.2', '2.0']) + eq(text.search_all('aba', '1.1'), ['1.2', '2.0', '1.0']) + + res = text.search_all('', '1.0') # empty pattern + eq(res[:5], ['1.0', '1.1', '1.2', '1.3', '1.4']) + eq(res[-5:], ['3.2', '3.3', '3.4', '3.5', '3.6']) + eq(text.search_all('nonexistent', '1.0'), []) + eq(text.search_all('-bit', '1.0'), ['3.2']) # starts with a hyphen + + eq(text.search_all('aba', '1.0', 'end', forwards=True), ['1.0', '2.0']) + eq(text.search_all('aba', 'end', '1.0', backwards=True), ['2.0', '1.2']) + + eq(text.search_all('aba', '1.0', overlap=True), ['1.0', '1.2', '2.0']) + eq(text.search_all('aba', 'end', '1.0', overlap=True, backwards=True), ['2.0', '1.2', '1.0']) + + eq(text.search_all('aba', '1.0', exact=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', exact=True), []) + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + + eq(text.search_all('ABA', '1.0'), []) + eq(text.search_all('ABA', '1.0', nocase=True), ['1.0', '2.0']) + + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', regexp=True, nolinestop=True), ['1.0', '1.4']) + + eq(text.search_all('aba', '1.0', '2.2'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', '2.2', strictlimits=True), ['1.0']) + eq(text.search_all('aba', '1.0', '2.3', strictlimits=True), ['1.0', '2.0']) + + var = tkinter.Variable(self.root) + eq(text.search_all('aba', '1.0', count=var), ['1.0', '2.0']) + self.assertEqual(var.get(), (3, 3) if self.wantobjects else '3 3') + + # TODO: Add test for elide=True def test_count(self): text = self.text diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c5453074039..737583a42c6 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4049,8 +4049,9 @@ def scan_dragto(self, x, y): self.tk.call(self._w, 'scan', 'dragto', x, y) def search(self, pattern, index, stopindex=None, - forwards=None, backwards=None, exact=None, - regexp=None, nocase=None, count=None, elide=None): + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, *, nolinestop=None, strictlimits=None): """Search PATTERN beginning from INDEX until STOPINDEX. Return the index of the first character of a match or an empty string.""" @@ -4062,12 +4063,39 @@ def search(self, pattern, index, stopindex=None, if nocase: args.append('-nocase') if elide: args.append('-elide') if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if strictlimits: args.append('-strictlimits') if pattern and pattern[0] == '-': args.append('--') args.append(pattern) args.append(index) - if stopindex: args.append(stopindex) + if stopindex is not None: args.append(stopindex) return str(self.tk.call(tuple(args))) + def search_all(self, pattern, index, stopindex=None, *, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, nolinestop=None, overlap=None, + strictlimits=None): + """Search all occurrences of PATTERN from INDEX to STOPINDEX. + Return a tuple of indices where matches begin.""" + args = [self._w, 'search', '-all'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if overlap: args.append('-overlap') + if strictlimits: args.append('-strictlimits') + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex is not None: args.append(stopindex) + result = self.tk.call(tuple(args)) + return self.tk.splitlist(result) + def see(self, index): """Scroll such that the character at INDEX is visible.""" self.tk.call(self._w, 'see', index) diff --git a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst new file mode 100644 index 00000000000..b175ab7cad4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst @@ -0,0 +1 @@ +Add support for ``-nolinestop``, and ``-strictlimits`` options to :meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` method for ``-all`` and ``-overlap`` options. From cc6b62ac561e857a2cc4eb4f43e1e0e9f53c09f1 Mon Sep 17 00:00:00 2001 From: Semyon Moroz Date: Mon, 17 Nov 2025 18:51:21 +0400 Subject: [PATCH 220/417] gh-130160: Add anchors to CLI Usage section for `cmdline` (#133182) --- Doc/library/cmdline.rst | 10 +++++----- Doc/library/ensurepip.rst | 4 +++- Doc/library/gzip.rst | 4 ++-- Doc/library/idle.rst | 4 +++- Doc/library/inspect.rst | 2 +- Doc/library/pdb.rst | 6 +++++- Doc/library/site.rst | 2 +- Doc/library/webbrowser.rst | 7 ++++++- 8 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7c..c43b10157f9 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -16,17 +16,17 @@ The following modules have a command-line interface. * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` @@ -52,8 +52,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 165b9a9f823..32b92c01570 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -42,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index cb36be42a83..d23c0741ddb 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -283,7 +283,7 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`gzip` module provides a simple command line interface to compress or @@ -296,7 +296,7 @@ Once executed the :mod:`gzip` module keeps the input file(s). Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 52e3726a0f5..a16f46ef812 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -661,7 +661,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 13a352cbdb2..c00db31a8ec 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1788,7 +1788,7 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`inspect` module also provides a basic introspection capability diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 90dc6648045..0bbdc425352 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,6 +76,10 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug .. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb You can also invoke :mod:`pdb` from the command line to debug other scripts. For @@ -334,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60e..d93e4dc7c75 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -270,7 +270,7 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index fd6abc70261..a2103d8fdd8 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -49,6 +49,11 @@ a new tab, with the browser being brought to the foreground. The use of the :mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If :mod:`ctypes` isn't available, calls to :func:`.open` will fail. +.. _webbrowser-cli: + +Command-line interface +---------------------- + .. program:: webbrowser The script :program:`webbrowser` can be used as a command-line interface for the @@ -232,7 +237,7 @@ Here are some simple examples:: .. _browser-controllers: -Browser Controller Objects +Browser controller objects -------------------------- Browser controllers provide the :attr:`~controller.name` attribute, From 274a26cca8e3d2f4de0283d4acbc80be391a5f6a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 17 Nov 2025 16:32:08 +0000 Subject: [PATCH 221/417] gh-135953: Simplify GC markers in the tachyon profiler (#141666) --- Lib/profiling/sampling/gecko_collector.py | 15 +++++++-------- Lib/test/test_profiling/test_sampling_profiler.py | 6 ++---- Modules/_remote_debugging_module.c | 11 ----------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 6c6700f1130..21c427b7c86 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -141,7 +141,6 @@ def collect(self, stack_frames): for thread_info in interpreter_info.threads: frames = thread_info.frame_info tid = thread_info.thread_id - gc_collecting = thread_info.gc_collecting # Initialize thread if needed if tid not in self.threads: @@ -197,16 +196,16 @@ def collect(self, stack_frames): self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), current_time, CATEGORY_GIL) - # Track GC events - attribute to all threads that hold the GIL during GC - # (GC is interpreter-wide but runs on whichever thread(s) have the GIL) - # If GIL switches during GC, multiple threads will get GC markers - if gc_collecting and has_gil: - # Start GC marker if not already started for this thread + # Track GC events by detecting frames in the stack trace + # This leverages the improved GC frame tracking from commit 336366fd7ca + # which precisely identifies the thread that initiated GC collection + has_gc_frame = any(frame[2] == "" for frame in frames) + if has_gc_frame: + # This thread initiated GC collection if tid not in self.gc_start_per_thread: self.gc_start_per_thread[tid] = current_time elif tid in self.gc_start_per_thread: - # End GC marker if it was running for this thread - # (either GC finished or thread lost GIL) + # End GC marker when no more GC frames are detected self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), current_time, CATEGORY_GC) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index a24dbb55cd7..2d00173c22c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -63,14 +63,13 @@ def __repr__(self): class MockThreadInfo: """Mock ThreadInfo for testing since the real one isn't accessible.""" - def __init__(self, thread_id, frame_info, status=0, gc_collecting=False): # Default to THREAD_STATE_RUNNING (0) + def __init__(self, thread_id, frame_info, status=0): # Default to THREAD_STATE_RUNNING (0) self.thread_id = thread_id self.frame_info = frame_info self.status = status - self.gc_collecting = gc_collecting def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status}, gc_collecting={self.gc_collecting})" + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" class MockInterpreterInfo: @@ -2742,7 +2741,6 @@ def __init__(self, thread_id, frame_info, status): self.thread_id = thread_id self.frame_info = frame_info self.status = status - self.gc_collecting = False # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 51b3c6bac02..6544e3a0ce6 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -186,7 +186,6 @@ static PyStructSequence_Field ThreadInfo_fields[] = { {"thread_id", "Thread ID"}, {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, {"frame_info", "Frame information"}, - {"gc_collecting", "Whether GC is collecting (interpreter-level)"}, {NULL} }; @@ -2726,8 +2725,6 @@ unwind_stack_for_thread( goto error; } - int gc_collecting = GET_MEMBER(int, gc_state, unwinder->debug_offsets.gc.collecting); - // Calculate thread status using flags (always) int status_flags = 0; @@ -2827,18 +2824,10 @@ unwind_stack_for_thread( goto error; } - PyObject *py_gc_collecting = PyBool_FromLong(gc_collecting); - if (py_gc_collecting == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create gc_collecting"); - Py_DECREF(py_status); - goto error; - } - // py_status contains status flags (bitfield) PyStructSequence_SetItem(result, 0, thread_id); PyStructSequence_SetItem(result, 1, py_status); // Steals reference PyStructSequence_SetItem(result, 2, frame_info); // Steals reference - PyStructSequence_SetItem(result, 3, py_gc_collecting); // Steals reference cleanup_stack_chunks(&chunks); return result; From 6b1bdf6c7a6c87f12a247a125e25f8e721cc731e Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya <141550576+XChaitanyaX@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:59:06 +0530 Subject: [PATCH 222/417] gh-141497: Make ipaddress.IP{v4,v6}Network.hosts() always returning an iterator (GH-141547) --- Lib/ipaddress.py | 4 +-- Lib/test/test_ipaddress.py | 34 +++++++++++++++++++ ...-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst | 4 +++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index aa0cf4a0620..f1062a8cd05 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1546,7 +1546,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == (self.max_prefixlen): - self.hosts = lambda: [IPv4Address(addr)] + self.hosts = lambda: iter((IPv4Address(addr),)) @property @functools.lru_cache() @@ -2337,7 +2337,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == self.max_prefixlen: - self.hosts = lambda: [IPv6Address(addr)] + self.hosts = lambda: iter((IPv6Address(addr),)) def hosts(self): """Generate Iterator over usable hosts in a network. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 11721a59972..3f017b97dc2 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -12,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -1472,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1494,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1503,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1512,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1520,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), diff --git a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst new file mode 100644 index 00000000000..328bfe067ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst @@ -0,0 +1,4 @@ +:mod:`ipaddress`: ensure that the methods +:meth:`IPv4Network.hosts() ` and +:meth:`IPv6Network.hosts() ` always return an +iterator. From 5d2eb98a91f2cd703d14f38c751ac7f52b2d7148 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 17 Nov 2025 18:47:00 +0100 Subject: [PATCH 223/417] gh-140578: Delete unnecessary NEWS entry (#141427) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .../2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst diff --git a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst deleted file mode 100644 index 702d38d4d24..00000000000 --- a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove outdated sencence in the documentation for :mod:`multiprocessing`, -that implied that :class:`concurrent.futures.ThreadPoolExecutor` did not -exist. From b3626321b6ebb46dd24acee2aa806450e70febfc Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 17 Nov 2025 14:40:47 -0500 Subject: [PATCH 224/417] gh-141004: Document `PyODict*` APIs (GH-141136) --- Doc/c-api/dict.rst | 89 ++++++++++++++++++++++++++++++++++++++++++ Doc/c-api/iterator.rst | 1 + 2 files changed, 90 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index b7f201811aa..ede1699cfeb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -477,3 +477,92 @@ Dictionary View Objects Return true if *op* is an instance of a dictionary items view. This function always succeeds. + + +Ordered Dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 7eaf72ec55f..bfbfe3c9279 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -108,6 +108,7 @@ Other Iterator Objects .. c:var:: PyTypeObject PyDictRevIterValue_Type .. c:var:: PyTypeObject PyDictIterItem_Type .. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type Type objects for iterators of various built-in objects. From 16ea9505ce690485bab38691e5a83f467757fc03 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:52:13 +0000 Subject: [PATCH 225/417] gh-141004: Document `Py_MEMCPY` (GH-141676) --- Doc/c-api/intro.rst | 8 ++++++++ Misc/NEWS.d/3.14.0a1.rst | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c76cc2f70ec..bace21b7981 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -183,6 +183,14 @@ complete listing. .. versionadded:: 3.6 +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. + Use :c:func:`!memcpy` directly instead. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + .. c:macro:: Py_MIN(x, y) Return the minimum value between ``x`` and ``y``. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 305a0b65b98..1938976fa42 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -6092,7 +6092,7 @@ Patch by Victor Stinner. .. nonce: qOr9GF .. section: C API -Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +Soft deprecate the :c:macro:`Py_MEMCPY` macro: use directly ``memcpy()`` instead. Patch by Victor Stinner. .. From 4867f717e21c3b5f0ad0e81f950c69dac6c95e6e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 18 Nov 2025 02:26:40 +0000 Subject: [PATCH 226/417] gh-140729: Fix subprocess handling in test_process_pool_executor_pickle (#141688) --- Lib/test/test_profiling/test_sampling_profiler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 2d00173c22c..c2cc2ddd48a 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3311,6 +3311,8 @@ def test_native_frames_disabled(self): self.assertNotIn("", output) +@requires_subprocess() +@skip_if_not_supported class TestProcessPoolExecutorSupport(unittest.TestCase): """ Test that ProcessPoolExecutor works correctly with profiling.sampling. @@ -3339,12 +3341,15 @@ def worker(x): "-d", "5", "-i", "100000", script, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as proc: - proc.wait(timeout=SHORT_TIMEOUT) - stdout = proc.stdout.read() - stderr = proc.stderr.read() + try: + stdout, stderr = proc.communicate(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() if "PermissionError" in stderr: self.skipTest("Insufficient permissions for remote profiling") From 58f3fe0d9b9882656e629e8caab687c7fcb21b36 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 18 Nov 2025 01:10:32 -0800 Subject: [PATCH 227/417] gh-129005: Remove copies from _pyio using take_bytes (#141539) Memory usage now matches that of _io for large files. --- Lib/_pyio.py | 8 ++++---- Lib/test/test_io/test_bufferedio.py | 3 ++- Lib/test/test_io/test_largefile.py | 6 ++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 423178e87a8..69a088df8fc 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -546,7 +546,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -620,7 +620,7 @@ def read(self, size=-1): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -628,7 +628,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -1738,7 +1738,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 30c34e818b1..3278665bdc9 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -1277,7 +1277,8 @@ def test_flush_and_readinto(self): def _readinto(bufio, n=-1): b = bytearray(n if n >= 0 else 9999) n = bufio.readinto(b) - return bytes(b[:n]) + b.resize(n) + return b.take_bytes() self.check_flush_and_read(_readinto) def test_flush_and_peek(self): diff --git a/Lib/test/test_io/test_largefile.py b/Lib/test/test_io/test_largefile.py index 41f7b70e5cf..438a90a92ed 100644 --- a/Lib/test/test_io/test_largefile.py +++ b/Lib/test/test_io/test_largefile.py @@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest): (i.e. > 2 GiB) files. """ - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, - # so memuse=2 is needed - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: @@ -154,7 +152,7 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_seek_readall(self, _size): # Seek which doesn't change position should readall successfully. with self.open(TESTFN, 'rb') as f: From 630cd37bfae0fc4021d9e9461b94d36e7ce6b95c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 18 Nov 2025 12:17:37 +0300 Subject: [PATCH 228/417] gh-141004: Document Py_HUGE_VAL/IS_FINITE/IS_INFINITE/IS_NAN (#141544) Co-authored-by: Victor Stinner Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/float.rst | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 79de5daaa90..b0d440580b9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -87,7 +87,7 @@ Floating-Point Objects ```` header. .. deprecated:: 3.15 - The macro is soft deprecated. + The macro is :term:`soft deprecated`. .. c:macro:: Py_NAN @@ -99,6 +99,14 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + + .. c:macro:: Py_MATH_E The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. @@ -147,6 +155,34 @@ Floating-Point Objects return PyFloat_FromDouble(copysign(INFINITY, sign)); +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- From b87613f21474ea848fec435cbfe63d8cb1c7c44c Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 18 Nov 2025 11:32:15 +0100 Subject: [PATCH 229/417] Add missing backticks in os and decimal docs (#141699) --- Doc/library/decimal.rst | 2 +- Doc/library/os.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 985153b5443..ba882f10bbe 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -264,7 +264,7 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. In accordance with the standard, the :mod:`decimal` module provides two ready to diff --git a/Doc/library/os.rst b/Doc/library/os.rst index dbc3c92c879..7dc6c177268 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -558,7 +558,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. From b420f6be53efdf40f552c94f19a7ce85f882b5e2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 18 Nov 2025 13:31:48 +0000 Subject: [PATCH 230/417] GH-139109: Support switch/case dispatch with the tracing interpreter. (GH-141703) --- .github/workflows/jit.yml | 26 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 9 +- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop_ids.h | 1 + Include/opcode_ids.h | 47 +- Lib/_opcode_metadata.py | 47 +- Python/bytecodes.c | 9 +- Python/ceval.c | 4 + Python/ceval_macros.h | 10 +- Python/executor_cases.c.h | 2 + Python/generated_cases.c.h | 113 +-- Python/instrumentation.c | 4 +- Python/opcode_targets.h | 910 +++++++++++----------- Python/optimizer_cases.c.h | 2 + Tools/cases_generator/analyzer.py | 7 +- Tools/cases_generator/target_generator.py | 4 +- Tools/cases_generator/tier1_generator.py | 2 +- 18 files changed, 617 insertions(+), 585 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3349eb04242..62325250bd3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,10 +57,9 @@ jobs: fail-fast: false matrix: target: -# To re-enable later when we support these. -# - i686-pc-windows-msvc/msvc -# - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -71,16 +70,15 @@ jobs: llvm: - 21 include: -# To re-enable later when we support these. -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-2022 -# - target: x86_64-pc-windows-msvc/msvc -# architecture: x64 -# runner: windows-2022 -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-11-arm + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-2022 + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-2022 + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c695..2fb46a6df50 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 548627dc798..cca88818c57 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -488,6 +488,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -971,6 +973,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1287,6 +1291,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1738,6 +1743,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1809,7 +1815,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2025,6 +2030,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2070,7 +2076,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 0307a174e77..e7177552cf6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -364,7 +364,7 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); -int +PyAPI_FUNC(int) _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 7a33a5b84fd..c38f28f9db1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -352,6 +352,7 @@ extern "C" { #define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE #define _TO_BOOL_STR 551 +#define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefc..0d066c16901 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -213,28 +213,29 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 207 #define UNPACK_SEQUENCE_TUPLE 208 #define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -249,7 +250,7 @@ extern "C" { #define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32..e681cb17e43 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -208,8 +208,9 @@ opmap = { 'CACHE': 0, 'RESERVED': 17, 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, + 'INSTRUMENTED_LINE': 253, + 'ENTER_EXECUTOR': 254, + 'TRACE_RECORD': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, 'CALL_FUNCTION_EX': 4, @@ -328,26 +329,26 @@ opmap = { 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, + 'INSTRUMENTED_END_FOR': 233, + 'INSTRUMENTED_POP_ITER': 234, + 'INSTRUMENTED_END_SEND': 235, + 'INSTRUMENTED_FOR_ITER': 236, + 'INSTRUMENTED_INSTRUCTION': 237, + 'INSTRUMENTED_JUMP_FORWARD': 238, + 'INSTRUMENTED_NOT_TAKEN': 239, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 240, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 241, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 243, + 'INSTRUMENTED_RESUME': 244, + 'INSTRUMENTED_RETURN_VALUE': 245, + 'INSTRUMENTED_YIELD_VALUE': 246, + 'INSTRUMENTED_END_ASYNC_FOR': 247, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 248, + 'INSTRUMENTED_CALL': 249, + 'INSTRUMENTED_CALL_KW': 250, + 'INSTRUMENTED_CALL_FUNCTION_EX': 251, + 'INSTRUMENTED_JUMP_BACKWARD': 252, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -362,4 +363,4 @@ opmap = { } HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 565eaa7a599..12ee506e4f2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5636,10 +5636,12 @@ dummy_func( DISPATCH(); } - label(record_previous_inst) { + inst(TRACE_RECORD, (--)) { #if _Py_TIER2 assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); @@ -5675,7 +5677,8 @@ dummy_func( } DISPATCH_GOTO_NON_TRACING(); #else - Py_FatalError("JIT label executed in non-jit build."); + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); #endif } diff --git a/Python/ceval.c b/Python/ceval.c index 25294ebd993..14fef42ea96 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1179,6 +1179,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#if !USE_COMPUTED_GOTOS + uint8_t tracing_mode = 0; + uint8_t dispatch_code; +#endif #endif _PyEntryFrame entry; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 05a2760671e..c30638c221a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,8 +134,8 @@ # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: -# define DISPATCH_GOTO() goto dispatch_opcode -# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -148,9 +148,9 @@ # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; #else -# define IS_JIT_TRACING() (0) -# define ENTER_TRACING() -# define LEAVE_TRACING() +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6796abf84ac..e1edd20b778 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7579,5 +7579,7 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0d4678df68c..b83b7c528e9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11,7 +11,7 @@ #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif { #endif /* _Py_TAIL_CALL_INTERP */ @@ -11683,6 +11683,68 @@ DISPATCH(); } + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + TARGET(UNARY_INVERT) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; @@ -12254,55 +12316,6 @@ JUMP_TO_LABEL(error); DISPATCH(); } - LABEL(record_previous_inst) - { - #if _Py_TIER2 - assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; - bool stop_tracing = (opcode == WITH_EXCEPT_START || - opcode == RERAISE || opcode == CLEANUP_THROW || - opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (full) { - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - DISPATCH_GOTO_NON_TRACING(); - } - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && - opcode == POP_TOP) || - (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && - opcode == STORE_FAST)) { - _tstate->jit_tracer_state.prev_state.instr_is_super = true; - } - else { - _tstate->jit_tracer_state.prev_state.instr = next_instr; - } - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _tstate->jit_tracer_state.prev_state.instr_frame = frame; - _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; - _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - (&next_instr[1])->counter = trigger_backoff_counter(); - } - DISPATCH_GOTO_NON_TRACING(); - #else - Py_FatalError("JIT label executed in non-jit build."); - #endif - } - LABEL(stop_tracing) { #if _Py_TIER2 diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 81e46a331e0..72b7433022f 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -191,7 +191,7 @@ is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return opcode != ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; + return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; } #ifndef NDEBUG @@ -526,7 +526,7 @@ valid_opcode(int opcode) if (IS_VALID_OPCODE(opcode) && opcode != CACHE && opcode != RESERVED && - opcode < 255) + opcode < 254) { return true; } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 1b9196503b5..b2fa7d01e8f 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -233,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -256,130 +255,131 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -387,88 +387,88 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -492,29 +492,29 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -528,7 +528,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -746,6 +745,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); @@ -983,6 +983,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, @@ -1023,234 +1024,234 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, - [BUILD_LIST] = _TAIL_CALL_record_previous_inst, - [BUILD_MAP] = _TAIL_CALL_record_previous_inst, - [BUILD_SET] = _TAIL_CALL_record_previous_inst, - [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_STRING] = _TAIL_CALL_record_previous_inst, - [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, - [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, - [CACHE] = _TAIL_CALL_record_previous_inst, - [CALL] = _TAIL_CALL_record_previous_inst, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, - [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, - [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, - [CALL_KW] = _TAIL_CALL_record_previous_inst, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, - [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, - [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, - [CALL_LEN] = _TAIL_CALL_record_previous_inst, - [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_STR_1] = _TAIL_CALL_record_previous_inst, - [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, - [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, - [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, - [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, - [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, - [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, - [COPY] = _TAIL_CALL_record_previous_inst, - [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, - [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, - [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, - [DELETE_FAST] = _TAIL_CALL_record_previous_inst, - [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [DELETE_NAME] = _TAIL_CALL_record_previous_inst, - [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [DICT_MERGE] = _TAIL_CALL_record_previous_inst, - [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, - [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [END_FOR] = _TAIL_CALL_record_previous_inst, - [END_SEND] = _TAIL_CALL_record_previous_inst, - [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, - [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, - [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, - [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, - [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, - [FOR_ITER] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, - [GET_AITER] = _TAIL_CALL_record_previous_inst, - [GET_ANEXT] = _TAIL_CALL_record_previous_inst, - [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, - [GET_ITER] = _TAIL_CALL_record_previous_inst, - [GET_LEN] = _TAIL_CALL_record_previous_inst, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, - [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, - [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, - [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, - [IS_OP] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, - [LOAD_CONST] = _TAIL_CALL_record_previous_inst, - [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, - [LOAD_NAME] = _TAIL_CALL_record_previous_inst, - [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, - [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, - [MAKE_CELL] = _TAIL_CALL_record_previous_inst, - [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, - [MAP_ADD] = _TAIL_CALL_record_previous_inst, - [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, - [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, - [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, - [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [NOP] = _TAIL_CALL_record_previous_inst, - [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, - [POP_ITER] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [POP_TOP] = _TAIL_CALL_record_previous_inst, - [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, - [PUSH_NULL] = _TAIL_CALL_record_previous_inst, - [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, - [RERAISE] = _TAIL_CALL_record_previous_inst, - [RESERVED] = _TAIL_CALL_record_previous_inst, - [RESUME] = _TAIL_CALL_record_previous_inst, - [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, - [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, - [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [SEND] = _TAIL_CALL_record_previous_inst, - [SEND_GEN] = _TAIL_CALL_record_previous_inst, - [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, - [SET_ADD] = _TAIL_CALL_record_previous_inst, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, - [SET_UPDATE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [STORE_DEREF] = _TAIL_CALL_record_previous_inst, - [STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [STORE_NAME] = _TAIL_CALL_record_previous_inst, - [STORE_SLICE] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [SWAP] = _TAIL_CALL_record_previous_inst, - [TO_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, - [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, - [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, - [UNARY_NOT] = _TAIL_CALL_record_previous_inst, - [UNPACK_EX] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, - [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, - [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [DELETE_NAME] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1281,6 +1282,5 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01263fe8c7a..9ebd113df2d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3483,3 +3483,5 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 */ + diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d39013db4f7..93aa4899fe6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1195,8 +1195,9 @@ def assign_opcodes( # This is an historical oddity. instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 - instmap["INSTRUMENTED_LINE"] = 254 - instmap["ENTER_EXECUTOR"] = 255 + instmap["INSTRUMENTED_LINE"] = 253 + instmap["ENTER_EXECUTOR"] = 254 + instmap["TRACE_RECORD"] = 255 instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] @@ -1221,7 +1222,7 @@ def assign_opcodes( # Specialized ops appear in their own section # Instrumented opcodes are at the end of the valid range min_internal = instmap["RESUME"] + 1 - min_instrumented = 254 - (len(instrumented) - 1) + min_instrumented = 254 - len(instrumented) assert min_internal + len(specialized) < min_instrumented next_opcode = 1 diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 36fa1d7fa49..f633f704485 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&record_previous_inst,\n" + targets[op] = f"&&TARGET_TRACE_RECORD,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: @@ -84,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 94ffb0118f0..c7ff5de681e 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -160,7 +160,7 @@ def generate_tier1( #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif {{ #endif /* _Py_TAIL_CALL_INTERP */ From f46785f8bc118e0efb840af1e520777b1baa03d9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:42:13 +0200 Subject: [PATCH 231/417] gh-133879: Copyedit "What's new in Python 3.15" (#141717) --- .../pending-removal-in-future.rst | 2 +- Doc/whatsnew/3.15.rst | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 7ed430625f3..30186741670 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -76,7 +76,7 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index cf5bef15203..24cc7e2d7eb 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -316,9 +316,7 @@ Other language changes and compression. Common code patterns which can be optimized with :func:`~bytearray.take_bytes` are listed below. - (Contributed by Cody Maloney in :gh:`139871`.) - - .. list-table:: Suggested Optimizing Refactors + .. list-table:: Suggested optimizing refactors :header-rows: 1 * - Description @@ -387,10 +385,12 @@ Other language changes buffer.resize(n) data = buffer.take_bytes() + (Contributed by Cody Maloney in :gh:`139871`.) + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, - and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass - the module name. It is needed to unambiguous :ref:`filter ` + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` syntax warnings by module name. (Contributed by Serhiy Storchaka in :gh:`135801`.) @@ -776,6 +776,17 @@ unittest (Contributed by Garry Cairns in :gh:`134567`.) +venv +---- + +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) + + warnings -------- @@ -788,17 +799,6 @@ warnings (Contributed by Serhiy Storchaka in :gh:`135801`.) -venv ----- - -* On POSIX platforms, platlib directories will be created if needed when - creating virtual environments, instead of using ``lib64 -> lib`` symlink. - This means purelib and platlib of virtual environments no longer share the - same ``lib`` directory on platforms where :data:`sys.platlibdir` is not - equal to ``lib``. - (Contributed by Rui Xi in :gh:`133951`.) - - xml.parsers.expat ----------------- @@ -1242,7 +1242,7 @@ Porting to Python 3.15 This section lists previously described changes and other bugfixes that may require changes to your code. -* :class:`sqlite3.Connection` APIs has been cleaned up. +* :class:`sqlite3.Connection` APIs have been cleaned up. * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` @@ -1262,7 +1262,7 @@ that may require changes to your code. * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. -* Resource warning is now emitted for unclosed +* A resource warning is now emitted for an unclosed :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. Use its :meth:`!close` method or the :func:`contextlib.closing` context manager to close it. From a62562859deea162a36dd5c99f0b87fe09af0292 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:50:49 +0200 Subject: [PATCH 232/417] Python 3.15.0a2 --- Doc/c-api/import.rst | 2 +- Doc/c-api/init.rst | 4 +- Doc/library/ast.rst | 2 +- Doc/library/decimal.rst | 2 +- Doc/library/functions.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/importlib.rst | 4 +- Doc/library/inspect.rst | 2 +- Doc/library/math.integer.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/os.rst | 14 +- Doc/library/stat.rst | 2 +- Doc/library/stdtypes.rst | 2 +- Doc/library/symtable.rst | 2 +- Doc/library/unicodedata.rst | 4 +- Doc/library/warnings.rst | 2 +- Doc/library/winreg.rst | 2 +- Doc/library/xml.etree.elementtree.rst | 2 +- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 51 +- Misc/NEWS.d/3.15.0a2.rst | 1746 +++++++++++++++++ ...-08-10-22-28-06.gh-issue-137618.FdNvIE.rst | 2 - ...-10-16-11-30-53.gh-issue-140189.YCrUyt.rst | 1 - ...-10-17-11-33-45.gh-issue-140239._k-GgW.rst | 1 - ...-10-22-12-44-07.gh-issue-140475.OhzQbR.rst | 1 - ...-10-25-08-07-06.gh-issue-140513.6OhLTs.rst | 2 - ...-10-29-12-30-38.gh-issue-140768.ITYrzw.rst | 1 - ...-10-31-13-20-16.gh-issue-140454.gF6dCe.rst | 3 - ...-10-06-22-17-47.gh-issue-139653.6-1MOd.rst | 4 - ...-10-15-15-59-59.gh-issue-140153.BO7sH4.rst | 2 - ...-10-26-16-45-06.gh-issue-140487.fGOqss.rst | 2 - ...-10-26-16-45-28.gh-issue-140556.s__Dae.rst | 2 - ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 - ...-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst | 1 - ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 - ...-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 - ...-11-10-11-26-26.gh-issue-141341.OsO6-y.rst | 2 - ...-06-24-13-12-58.gh-issue-134786.MF0VVk.rst | 2 - ...-07-08-00-41-46.gh-issue-136327.7AiTb_.rst | 2 - ...-07-29-17-51-14.gh-issue-131253.GpRjWy.rst | 1 - ...-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst | 2 - ...-09-15-13-06-11.gh-issue-138944.PeCgLb.rst | 3 - ...-09-23-21-01-12.gh-issue-139269.1rIaxy.rst | 1 - ...-10-03-17-51-43.gh-issue-139475._684ED.rst | 2 - ...-10-06-10-03-37.gh-issue-139640.gY5oTb.rst | 3 - ...10-06-10-03-37.gh-issue-139640.gY5oTb2.rst | 3 - ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 - ...-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst | 2 - ...-10-13-13-54-19.gh-issue-139914.M-y_3E.rst | 1 - ...-10-14-17-07-37.gh-issue-140067.ID2gOm.rst | 1 - ...-10-14-18-24-16.gh-issue-139871.SWtuUz.rst | 2 - ...-10-14-20-18-31.gh-issue-140080.8ROjxW.rst | 1 - ...-10-15-00-21-40.gh-issue-140061.J0XeDV.rst | 2 - ...-10-15-17-12-32.gh-issue-140149.cy1m3d.rst | 2 - ...-10-16-21-47-00.gh-issue-140104.A8SQIm.rst | 2 - ...-10-17-14-38-10.gh-issue-140253.gCqFaL.rst | 2 - ...-10-17-18-03-12.gh-issue-139951.IdwM2O.rst | 7 - ...-10-17-20-23-19.gh-issue-140257.8Txmem.rst | 2 - ...-10-18-18-08-36.gh-issue-140301.m-2HxC.rst | 1 - ...-10-18-19-52-20.gh-issue-116738.NLJW0L.rst | 2 - ...-10-18-21-29-45.gh-issue-140306.xS5CcS.rst | 2 - ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 - ...-10-19-10-32-28.gh-issue-136895.HfsEh0.rst | 1 - ...-10-20-11-24-36.gh-issue-140358.UQuKdV.rst | 4 - ...-10-21-06-51-50.gh-issue-140406.0gJs8M.rst | 2 - ...-10-21-09-20-03.gh-issue-140398.SoABwJ.rst | 4 - ...-10-22-11-30-16.gh-issue-135904.3WE5oW.rst | 3 - ...-10-22-12-48-05.gh-issue-140476.F3-d1P.rst | 2 - ...-10-22-17-22-22.gh-issue-140431.m8D_A-.rst | 3 - ...-10-22-23-26-37.gh-issue-140443.wT5i1A.rst | 5 - ...-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst | 2 - ...-10-24-14-29-12.gh-issue-133467.A5d6TM.rst | 1 - ...-10-24-20-16-42.gh-issue-140517.cqun-K.rst | 3 - ...-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 - ...-10-25-07-25-52.gh-issue-140544.lwjtQe.rst | 1 - ...-10-25-17-36-46.gh-issue-140576.kj0SCY.rst | 2 - ...-10-25-21-31-43.gh-issue-131527.V-JVNP.rst | 2 - ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 - ...-10-29-20-59-10.gh-issue-140373.-uoaPP.rst | 2 - ...5-10-31-14-03-42.gh-issue-90344.gvZigO.rst | 1 - ...-11-02-12-47-38.gh-issue-140530.S934bp.rst | 2 - ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 - ...-11-03-17-21-38.gh-issue-140939.FVboAw.rst | 2 - ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 - ...-11-04-12-18-06.gh-issue-140942.GYns6n.rst | 2 - ...-11-05-19-50-37.gh-issue-140643.QCEOqG.rst | 3 - ...-11-10-23-07-06.gh-issue-141312.H-58GB.rst | 2 - ...-11-11-13-40-45.gh-issue-141367.I5KY7F.rst | 2 - ...-11-14-00-19-45.gh-issue-141528.VWdax1.rst | 3 - ...-11-14-16-25-15.gh-issue-114203.n3tlQO.rst | 1 - ...-11-15-01-21-00.gh-issue-141579.aB7cD9.rst | 2 - ...9-06-02-13-56-16.gh-issue-81313.axawSH.rst | 1 - ...-03-21-10-59-40.gh-issue-102431.eUDnf4.rst | 2 - ...-05-28-17-14-30.gh-issue-119668.RrIGpn.rst | 1 - ...-06-26-16-16-43.gh-issue-121011.qW54eh.rst | 2 - ...-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst | 4 - ...-03-04-17-19-26.gh-issue-130693.Kv01r8.rst | 1 - ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 - ...-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst | 2 - ...-05-07-22-09-28.gh-issue-133601.9kUL3P.rst | 1 - ...-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst | 1 - ...-06-10-18-02-29.gh-issue-135307.fXGrcK.rst | 2 - ...-06-29-22-01-00.gh-issue-133390.I1DW_3.rst | 2 - ...-07-01-04-57-57.gh-issue-136057.4-t596.rst | 1 - ...5-07-14-09-33-17.gh-issue-55531.Gt2e12.rst | 4 - ...-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst | 1 - ...5-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst | 2 - ...-08-26-08-17-56.gh-issue-138151.I6CdAk.rst | 3 - ...-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst | 2 - ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 - ...-09-11-15-03-37.gh-issue-138775.w7rnSx.rst | 2 - ...-09-12-09-34-37.gh-issue-138764.mokHoY.rst | 3 - ...-09-13-12-19-17.gh-issue-138859.PxjIoN.rst | 1 - ...-09-15-21-03-11.gh-issue-138891.oZFdtR.rst | 2 - ...5-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst | 2 - ...-09-23-09-46-46.gh-issue-139246.pzfM-w.rst | 1 - ...-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst | 3 - ...5-09-30-12-52-54.gh-issue-63161.mECM1A.rst | 3 - ...-10-02-22-29-00.gh-issue-139462.VZXUHe.rst | 3 - ...-10-11-09-07-06.gh-issue-139940.g54efZ.rst | 1 - ...-10-13-11-25-41.gh-issue-136702.uvLGK1.rst | 3 - ...5-10-14-20-27-06.gh-issue-76007.2NcUbo.rst | 2 - ...-10-15-02-26-50.gh-issue-140135.54JYfM.rst | 2 - ...-10-15-15-10-34.gh-issue-140166.NtxRez.rst | 1 - ...-10-15-17-23-51.gh-issue-140141.j2mUDB.rst | 5 - ...-10-15-20-47-04.gh-issue-140120.3gffZq.rst | 2 - ...-10-15-21-42-13.gh-issue-140041._Fka2j.rst | 1 - ...-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst | 2 - ...-10-16-17-17-20.gh-issue-135801.faH3fa.rst | 6 - ...-10-16-22-49-16.gh-issue-140212.llBNd0.rst | 5 - ...-10-17-12-33-01.gh-issue-140251.esM-OX.rst | 1 - ...-10-17-20-42-38.gh-issue-129117.X9jr4p.rst | 3 - ...-10-17-23-58-11.gh-issue-140272.lhY8uS.rst | 1 - ...5-10-18-14-30-21.gh-issue-76007.peEgcr.rst | 1 - ...5-10-18-15-20-25.gh-issue-76007.SNUzRq.rst | 2 - ...-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst | 3 - ...-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst | 1 - ...-10-22-12-56-57.gh-issue-140448.GsEkXD.rst | 2 - ...-10-22-20-52-13.gh-issue-140474.xIWlip.rst | 2 - ...-10-23-12-12-22.gh-issue-138774.mnh2gU.rst | 2 - ...-10-23-13-42-15.gh-issue-140481.XKxWpq.rst | 1 - ...-10-23-19-39-16.gh-issue-138162.Znw5DN.rst | 2 - ...-10-25-21-04-00.gh-issue-140607.oOZGxS.rst | 2 - ...-10-25-21-26-16.gh-issue-140593.OxlLc9.rst | 3 - ...-10-25-22-55-07.gh-issue-140601.In3MlS.rst | 4 - ...-10-26-16-24-12.gh-issue-140633.ioayC1.rst | 2 - ...-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst | 3 - ...-10-27-13-49-31.gh-issue-140634.ULng9G.rst | 1 - ...-10-27-16-01-41.gh-issue-125434.qy0uRA.rst | 2 - ...-10-27-18-29-42.gh-issue-140590.LT9HHn.rst | 2 - ...-10-28-02-46-56.gh-issue-139946.aN3_uY.rst | 1 - ...-10-28-17-43-51.gh-issue-140228.8kfHhO.rst | 1 - ...-10-29-09-40-10.gh-issue-140741.L13UCV.rst | 2 - ...-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst | 1 - ...-10-29-16-53-00.gh-issue-140766.CNagKF.rst | 1 - ...-10-30-12-36-19.gh-issue-140790._3T6-N.rst | 1 - ...-10-30-15-33-07.gh-issue-137821.8_Iavt.rst | 2 - ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 - ...-10-31-15-06-26.gh-issue-140691.JzHGtg.rst | 3 - ...-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst | 1 - ...-11-01-00-34-53.gh-issue-140826.JEDd7U.rst | 2 - ...-11-01-00-36-14.gh-issue-140874.eAWt3K.rst | 1 - ...-11-01-14-44-09.gh-issue-140873.kfuc9B.rst | 2 - ...-11-02-09-37-22.gh-issue-140734.f8gST9.rst | 2 - ...-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst | 3 - ...-11-02-19-23-32.gh-issue-140815.McEG-T.rst | 2 - ...-11-03-05-38-31.gh-issue-125115.jGS8MN.rst | 1 - ...-11-03-16-23-54.gh-issue-140797.DuFEeR.rst | 2 - ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 - ...-11-04-15-40-35.gh-issue-137969.9VZQVt.rst | 3 - ...-11-04-20-08-41.gh-issue-141018.d_oyOI.rst | 2 - ...-11-06-15-11-50.gh-issue-141141.tgIfgH.rst | 1 - ...5-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 - ...5-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 - ...-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 - ...-11-10-01-47-18.gh-issue-141314.baaa28.rst | 1 - ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 - ...-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 - ...-11-13-14-51-30.gh-issue-140938.kXsHHv.rst | 2 - ...-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst | 4 - ...-05-30-22-33-27.gh-issue-136065.bu337o.rst | 1 - ...-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst | 2 - ...-08-15-23-08-44.gh-issue-137836.b55rhh.rst | 3 - ...-07-09-21-45-51.gh-issue-136442.jlbklP.rst | 1 - ...-10-15-00-52-12.gh-issue-140082.fpET50.rst | 3 - ...-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst | 1 - ...-09-20-20-31-54.gh-issue-139188.zfcxkW.rst | 1 - ...-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst | 1 - ...-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst | 2 - ...-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 - ...-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst | 1 - README.rst | 2 +- 192 files changed, 1811 insertions(+), 394 deletions(-) create mode 100644 Misc/NEWS.d/3.15.0a2.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 24e673d3d13..971c6a69e5d 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -353,4 +353,4 @@ Importing Modules On error, return NULL with an exception set. - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 18ee1611807..3cac2c8b213 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1390,7 +1390,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) @@ -1400,7 +1400,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 0ea3c3c59a6..2e7d0dbc26e 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2261,7 +2261,7 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index ba882f10bbe..05937775699 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1575,7 +1575,7 @@ Constants Specification that this implementation complies with. See https://speleotrove.com/decimal/decarith.html for the specification. - .. versionadded:: next + .. versionadded:: 3.15 The following constants are only relevant for the C module. They diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3257daf89d3..8314fed80fa 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -377,7 +377,7 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index b2e2e11c0dc..97136b23408 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -716,7 +716,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 - .. versionchanged:: next + .. versionchanged:: 3.15 Added support of non-:term:`descriptor` callables. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 03ba23b6216..3f0a54ac535 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -480,7 +480,7 @@ ABC hierarchy:: .. versionchanged:: 3.5 Made the method static. - .. versionadded:: next + .. versionadded:: 3.15 Added the *fullname* parameter. @@ -1048,7 +1048,7 @@ find and load modules. :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, forcing the path value to be recomputed next time it is accessed. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: SourceFileLoader(fullname, path) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index c00db31a8ec..5220c559d3d 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -636,7 +636,7 @@ Retrieving source code .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. - .. versionchanged:: next + .. versionchanged:: 3.15 Added parameters *inherit_class_doc* and *fallback_to_class_doc*. Documentation strings on :class:`~functools.cached_property` diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst index 6a9fe74c5e8..0068ae2bdd5 100644 --- a/Doc/library/math.integer.rst +++ b/Doc/library/math.integer.rst @@ -4,7 +4,7 @@ .. module:: math.integer :synopsis: Integer-specific mathematics functions. -.. versionadded:: next +.. versionadded:: 3.15 -------------- diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 54c98346b27..186f99e9591 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,7 +781,7 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: next +.. deprecated:: 3.15 These aliases are :term:`soft deprecated` in favor of the :mod:`math.integer` functions. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7dc6c177268..671270d6112 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3404,7 +3404,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: statx_result @@ -3661,7 +3661,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: STATX_TYPE @@ -3690,7 +3690,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_FORCE_SYNC @@ -3700,7 +3700,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_DONT_SYNC @@ -3709,7 +3709,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_SYNC_AS_STAT @@ -3721,7 +3721,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_NO_AUTOMOUNT @@ -3733,7 +3733,7 @@ features: .. availability:: Linux. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: statvfs(path) diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 1cbec3ab847..82012b31a00 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -511,4 +511,4 @@ meaning of these constants. STATX_ATTR_DAX STATX_ATTR_WRITE_ATOMIC - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c539345e598..3bcaba0b3e1 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3191,7 +3191,7 @@ objects. Taking all bytes is a zero-copy operation. - .. versionadded:: next + .. versionadded:: 3.15 See the :ref:`What's New ` entry for common code patterns which can be optimized with diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index c0d9e79197d..f5e6f9f8acf 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -30,7 +30,7 @@ Generating Symbol Tables It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index fd5f56bd7ea..34f21f49b4b 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -156,7 +156,7 @@ following functions: >>> unicodedata.isxidstart('0') False - .. versionadded:: next + .. versionadded:: 3.15 .. function:: isxidcontinue(chr, /) @@ -171,7 +171,7 @@ following functions: >>> unicodedata.isxidcontinue(' ') False - .. versionadded:: next + .. versionadded:: 3.15 .. function:: decomposition(chr, /) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 2f3cf6008f5..0de7a90bfcb 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -513,7 +513,7 @@ Available Functions .. versionchanged:: 3.6 Add the *source* parameter. - .. versionchanged:: next + .. versionchanged:: 3.15 If no module is passed, test the filter regular expression against module names created from the path, not only the path itself. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index b150c53735d..89def6e2afe 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -818,6 +818,6 @@ integer handle, and also disconnect the Windows handle from the handle object. will automatically close *key* when control leaves the :keyword:`with` block. -.. versionchanged:: next +.. versionchanged:: 3.15 Handle objects are now compared by their underlying Windows handle value instead of object identity for equality comparisons. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index cbbc87b4721..e59759683a6 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -656,7 +656,7 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. - .. versionchanged:: next + .. versionchanged:: 3.15 A :exc:`ResourceWarning` is now emitted if the iterator opened a file and is not explicitly closed. diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e3996ee8679..899c892631f 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,10 +24,10 @@ #define PY_MINOR_VERSION 15 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.15.0a1+" +#define PY_VERSION "3.15.0a2" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 293c3189589..11ffc6bf3a1 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Oct 14 13:46:01 2025 +# Autogenerated by Sphinx on Tue Nov 18 16:51:09 2025 # as part of the release process. topics = { @@ -1098,10 +1098,10 @@ class and instance attributes applies as for regular assignments. 'bltin-ellipsis-object': r'''The Ellipsis Object ******************* -This object is commonly used used to indicate that something is -omitted. It supports no special operations. There is exactly one -ellipsis object, named "Ellipsis" (a built-in name). -"type(Ellipsis)()" produces the "Ellipsis" singleton. +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +"Ellipsis" singleton. It is written as "Ellipsis" or "...". @@ -4140,6 +4140,10 @@ def double(x): available for commands and command arguments, e.g. the current global and local names are offered as arguments of the "p" command. + +Command-line interface +====================== + You can also invoke "pdb" from the command line to debug other scripts. For example: @@ -4155,7 +4159,7 @@ def double(x): -c, --command To execute commands as if given in a ".pdbrc" file; see Debugger - Commands. + commands. Changed in version 3.2: Added the "-c" option. @@ -4376,7 +4380,7 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa See the documentation for the functions explained above. -Debugger Commands +Debugger commands ================= The commands recognized by the debugger are listed below. Most @@ -5616,9 +5620,8 @@ class of the instance or a *non-virtual base class* thereof. The 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric -parts. For example, "077.010" is legal, and denotes the same number as -"77.10". +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". As in integer literals, single underscores may occur between digits to help readability: @@ -7435,9 +7438,8 @@ class body. A "SyntaxError" is raised if a variable is used or 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric -parts. For example, "077.010" is legal, and denotes the same number as -"77.10". +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". As in integer literals, single underscores may occur between digits to help readability: @@ -7685,9 +7687,8 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is ************************* *Objects* are Python’s abstraction for data. All data in a Python -program is represented by objects or by relations between objects. (In -a sense, and in conformance to Von Neumann’s model of a “stored -program computer”, code is also represented by objects.) +program is represented by objects or by relations between objects. +Even code is represented by objects. Every object has an identity, a type and a value. An object’s *identity* never changes once it has been created; you may think of it @@ -10301,6 +10302,17 @@ class is used in a class pattern with positional arguments, each follow uncased characters and lowercase characters only cased ones. Return "False" otherwise. + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + str.isupper() Return "True" if all cased characters [4] in the string are @@ -10663,6 +10675,8 @@ class is used in a class pattern with positional arguments, each >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also "istitle()". + str.translate(table, /) Return a copy of the string in which each character has been mapped @@ -12362,6 +12376,11 @@ class method object, it is transformed into an instance method object | | "X.__bases__" will be exactly equal to "(A, B, | | | C)". | +----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ | type.__doc__ | The class’s documentation string, or "None" if | | | undefined. Not inherited by subclasses. | +----------------------------------------------------+----------------------------------------------------+ diff --git a/Misc/NEWS.d/3.15.0a2.rst b/Misc/NEWS.d/3.15.0a2.rst new file mode 100644 index 00000000000..ba82c854fac --- /dev/null +++ b/Misc/NEWS.d/3.15.0a2.rst @@ -0,0 +1,1746 @@ +.. date: 2025-11-04-19-20-05 +.. gh-issue: 140849 +.. nonce: YjB2ZZ +.. release date: 2025-11-18 +.. section: Windows + +Update bundled liblzma to version 5.8.1. + +.. + +.. date: 2025-11-12-12-54-28 +.. gh-issue: 141442 +.. nonce: 50dS3P +.. section: Tools/Demos + +The iOS testbed now correctly handles test arguments that contain spaces. + +.. + +.. date: 2025-10-29-15-20-19 +.. gh-issue: 140702 +.. nonce: ZXtW8h +.. section: Tools/Demos + +The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment +variable to iOS apps being tested. + +.. + +.. date: 2025-09-21-10-30-08 +.. gh-issue: 139198 +.. nonce: Fm7NfU +.. section: Tools/Demos + +Remove ``Tools/scripts/checkpip.py`` script. + +.. + +.. date: 2025-09-20-20-31-54 +.. gh-issue: 139188 +.. nonce: zfcxkW +.. section: Tools/Demos + +Remove ``Tools/tz/zdump.py`` script. + +.. + +.. date: 2025-10-23-16-39-49 +.. gh-issue: 140482 +.. nonce: ZMtyeD +.. section: Tests + +Preserve and restore the state of ``stty echo`` as part of the test +environment. + +.. + +.. date: 2025-10-15-00-52-12 +.. gh-issue: 140082 +.. nonce: fpET50 +.. section: Tests + +Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color +enabled so that :mod:`unittest` which is run by it with redirected output +will output in color. + +.. + +.. date: 2025-07-09-21-45-51 +.. gh-issue: 136442 +.. nonce: jlbklP +.. section: Tests + +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` +raises an exception + +.. + +.. date: 2025-08-15-23-08-44 +.. gh-issue: 137836 +.. nonce: b55rhh +.. section: Security + +Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", +"noembed" and "noframes", and optionally RAWTEXT element "noscript" in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-06-28-13-23-53 +.. gh-issue: 136063 +.. nonce: aGk0Jv +.. section: Security + +:mod:`email.message`: ensure linear complexity for legacy HTTP parameters +parsing. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-30-22-33-27 +.. gh-issue: 136065 +.. nonce: bu337o +.. section: Security + +Fix quadratic complexity in :func:`os.path.expandvars`. + +.. + +.. date: 2025-11-14-16-24-20 +.. gh-issue: 141497 +.. nonce: L_CxDJ +.. section: Library + +:mod:`ipaddress`: ensure that the methods :meth:`IPv4Network.hosts() +` and :meth:`IPv6Network.hosts() +` always return an iterator. + +.. + +.. date: 2025-11-13-14-51-30 +.. gh-issue: 140938 +.. nonce: kXsHHv +.. section: Library + +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now +raise a :exc:`ValueError` when the input contains an infinity or a NaN. + +.. + +.. date: 2025-11-12-15-42-47 +.. gh-issue: 124111 +.. nonce: hTw4OE +.. section: Library + +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. + +.. + +.. date: 2025-11-12-01-49-03 +.. gh-issue: 137109 +.. nonce: D6sq2B +.. section: Library + +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. + +.. + +.. date: 2025-11-10-01-47-18 +.. gh-issue: 141314 +.. nonce: baaa28 +.. section: Library + +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files +with standalone carriage return (``\r``) line endings. + +.. + +.. date: 2025-11-09-18-55-13 +.. gh-issue: 141311 +.. nonce: qZ3swc +.. section: Library + +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. + +.. + +.. date: 2025-11-08-13-03-10 +.. gh-issue: 87710 +.. nonce: XJeZlP +.. section: Library + +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. + +.. + +.. date: 2025-11-07-12-25-46 +.. gh-issue: 85524 +.. nonce: 9SWFIC +.. section: Library + +Update ``io.FileIO.readall``, an implementation of +:meth:`io.RawIOBase.readall`, to follow :class:`io.IOBase` guidelines and +raise :exc:`io.UnsupportedOperation` when a file is in "w" mode rather than +:exc:`OSError` + +.. + +.. date: 2025-11-06-15-11-50 +.. gh-issue: 141141 +.. nonce: tgIfgH +.. section: Library + +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by +Benel Tayar. + +.. + +.. date: 2025-11-04-20-08-41 +.. gh-issue: 141018 +.. nonce: d_oyOI +.. section: Library + +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. + +.. + +.. date: 2025-11-04-15-40-35 +.. gh-issue: 137969 +.. nonce: 9VZQVt +.. section: Library + +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. + +.. + +.. date: 2025-11-04-12-16-13 +.. gh-issue: 75593 +.. nonce: EFVhKR +.. section: Library + +Add support of :term:`path-like objects ` and +:term:`bytes-like objects ` in :func:`wave.open`. + +.. + +.. date: 2025-11-03-16-23-54 +.. gh-issue: 140797 +.. nonce: DuFEeR +.. section: Library + +The undocumented :class:`!re.Scanner` class now forbids regular expressions +containing capturing groups in its lexicon patterns. Patterns using +capturing groups could previously lead to crashes with segmentation fault. +Use non-capturing groups (?:...) instead. + +.. + +.. date: 2025-11-03-05-38-31 +.. gh-issue: 125115 +.. nonce: jGS8MN +.. section: Library + +Refactor the :mod:`pdb` parsing issue so positional arguments can pass +through intuitively. + +.. + +.. date: 2025-11-02-19-23-32 +.. gh-issue: 140815 +.. nonce: McEG-T +.. section: Library + +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. + +.. + +.. date: 2025-11-02-11-46-00 +.. gh-issue: 100218 +.. nonce: 9Ezfdq +.. section: Library + +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-11-02-09-37-22 +.. gh-issue: 140734 +.. nonce: f8gST9 +.. section: Library + +:mod:`multiprocessing`: fix off-by-one error when checking the length of a +temporary socket file path. Patch by Bénédikt Tran. + +.. + +.. date: 2025-11-01-14-44-09 +.. gh-issue: 140873 +.. nonce: kfuc9B +.. section: Library + +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. + +.. + +.. date: 2025-11-01-00-36-14 +.. gh-issue: 140874 +.. nonce: eAWt3K +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.3 + +.. + +.. date: 2025-11-01-00-34-53 +.. gh-issue: 140826 +.. nonce: JEDd7U +.. section: Library + +Now :class:`!winreg.HKEYType` objects are compared by their underlying +Windows registry handle value instead of their object identity. + +.. + +.. date: 2025-10-31-16-25-13 +.. gh-issue: 140808 +.. nonce: XBiQ4j +.. section: Library + +The internal class ``mailbox._ProxyFile`` is no longer a parameterized +generic. + +.. + +.. date: 2025-10-31-15-06-26 +.. gh-issue: 140691 +.. nonce: JzHGtg +.. section: Library + +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. + +.. + +.. date: 2025-10-31-13-57-55 +.. gh-issue: 103847 +.. nonce: VM7TnW +.. section: Library + +Fix hang when cancelling process created by +:func:`asyncio.create_subprocess_exec` or +:func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-30-15-33-07 +.. gh-issue: 137821 +.. nonce: 8_Iavt +.. section: Library + +Convert ``_json`` module to use Argument Clinic. Patched by Yoonho Hann. + +.. + +.. date: 2025-10-30-12-36-19 +.. gh-issue: 140790 +.. nonce: _3T6-N +.. section: Library + +Initialize all Pdb's instance variables in ``__init__``, remove some +hasattr/getattr + +.. + +.. date: 2025-10-29-16-53-00 +.. gh-issue: 140766 +.. nonce: CNagKF +.. section: Library + +Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. + +.. + +.. date: 2025-10-29-16-12-41 +.. gh-issue: 120057 +.. nonce: qGj5Dl +.. section: Library + +Add :func:`os.reload_environ` to ``os.__all__``. + +.. + +.. date: 2025-10-29-09-40-10 +.. gh-issue: 140741 +.. nonce: L13UCV +.. section: Library + +Fix :func:`profiling.sampling.sample` incorrectly handling a +:exc:`FileNotFoundError` or :exc:`PermissionError`. + +.. + +.. date: 2025-10-28-17-43-51 +.. gh-issue: 140228 +.. nonce: 8kfHhO +.. section: Library + +Avoid making unnecessary filesystem calls for frozen modules in +:mod:`linecache` when the global module cache is not present. + +.. + +.. date: 2025-10-28-02-46-56 +.. gh-issue: 139946 +.. nonce: aN3_uY +.. section: Library + +Error and warning keywords in ``argparse.ArgumentParser`` messages are now +colorized when color output is enabled, fixing a visual inconsistency in +which they remained plain text while other output was colorized. + +.. + +.. date: 2025-10-27-18-29-42 +.. gh-issue: 140590 +.. nonce: LT9HHn +.. section: Library + +Fix arguments checking for the :meth:`!functools.partial.__setstate__` that +may lead to internal state corruption and crash. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-27-16-01-41 +.. gh-issue: 125434 +.. nonce: qy0uRA +.. section: Library + +Display thread name in :mod:`faulthandler` on Windows. Patch by Victor +Stinner. + +.. + +.. date: 2025-10-27-13-49-31 +.. gh-issue: 140634 +.. nonce: ULng9G +.. section: Library + +Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. + +.. + +.. date: 2025-10-27-00-40-49 +.. gh-issue: 140650 +.. nonce: DYJPJ9 +.. section: Library + +Fix an issue where closing :class:`io.BufferedWriter` could crash if the +closed attribute raised an exception on access or could not be converted to +a boolean. + +.. + +.. date: 2025-10-26-16-24-12 +.. gh-issue: 140633 +.. nonce: ioayC1 +.. section: Library + +Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute +when loading an extension module packaged as Apple Framework. + +.. + +.. date: 2025-10-25-22-55-07 +.. gh-issue: 140601 +.. nonce: In3MlS +.. section: Library + +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. + +.. + +.. date: 2025-10-25-21-26-16 +.. gh-issue: 140593 +.. nonce: OxlLc9 +.. section: Library + +:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom +element declaration handler. Patch by Sebastian Pipping. + +.. + +.. date: 2025-10-25-21-04-00 +.. gh-issue: 140607 +.. nonce: oOZGxS +.. section: Library + +Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned +by :meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). + +.. + +.. date: 2025-10-23-19-39-16 +.. gh-issue: 138162 +.. nonce: Znw5DN +.. section: Library + +Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the +*extra* argument. + +.. + +.. date: 2025-10-23-13-42-15 +.. gh-issue: 140481 +.. nonce: XKxWpq +.. section: Library + +Improve error message when trying to iterate a Tk widget, image or font. + +.. + +.. date: 2025-10-23-12-12-22 +.. gh-issue: 138774 +.. nonce: mnh2gU +.. section: Library + +:func:`ast.unparse` now generates full source code when handling +:class:`ast.Interpolation` nodes that do not have a specified source. + +.. + +.. date: 2025-10-22-20-52-13 +.. gh-issue: 140474 +.. nonce: xIWlip +.. section: Library + +Fix memory leak in :class:`array.array` when creating arrays from an empty +:class:`str` and the ``u`` type code. + +.. + +.. date: 2025-10-22-12-56-57 +.. gh-issue: 140448 +.. nonce: GsEkXD +.. section: Library + +Change the default of ``suggest_on_error`` to ``True`` in +``argparse.ArgumentParser``. + +.. + +.. date: 2025-10-21-15-54-13 +.. gh-issue: 137530 +.. nonce: ZyIVUH +.. section: Library + +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by +replacing the annotations that were in-line in the generated source code +with ``__annotate__`` functions attached to the methods. + +.. + +.. date: 2025-10-20-12-33-49 +.. gh-issue: 140348 +.. nonce: SAKnQZ +.. section: Library + +Fix regression in Python 3.14.0 where using the ``|`` operator on a +:class:`typing.Union` object combined with an object that is not a type +would raise an error. + +.. + +.. date: 2025-10-18-15-20-25 +.. gh-issue: 76007 +.. nonce: SNUzRq +.. section: Library + +:mod:`decimal`: Deprecate ``__version__`` and replace with +:data:`decimal.SPEC_VERSION`. + +.. + +.. date: 2025-10-18-14-30-21 +.. gh-issue: 76007 +.. nonce: peEgcr +.. section: Library + +Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. + +.. + +.. date: 2025-10-17-23-58-11 +.. gh-issue: 140272 +.. nonce: lhY8uS +.. section: Library + +Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. + +.. + +.. date: 2025-10-17-20-42-38 +.. gh-issue: 129117 +.. nonce: X9jr4p +.. section: Library + +:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and +:func:`~unicodedata.isxidcontinue` functions to check whether a character +can start or continue a `Unicode Standard Annex #31 +`_ identifier. + +.. + +.. date: 2025-10-17-12-33-01 +.. gh-issue: 140251 +.. nonce: esM-OX +.. section: Library + +Colorize the default import statement ``import asyncio`` in asyncio REPL. + +.. + +.. date: 2025-10-16-22-49-16 +.. gh-issue: 140212 +.. nonce: llBNd0 +.. section: Library + +Calendar's HTML formatting now accepts year and month as options. +Previously, running ``python -m calendar -t html 2025 10`` would result in +an error message. It now generates an HTML document displaying the calendar +for the specified month. Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-16-17-17-20 +.. gh-issue: 135801 +.. nonce: faH3fa +.. section: Library + +Improve filtering by module in :func:`warnings.warn_explicit` if no *module* +argument is passed. It now tests the module regular expression in the +warnings filter not only against the filename with ``.py`` stripped, but +also against module names constructed starting from different parent +directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, +``.pyw`` stripped). + +.. + +.. date: 2025-10-16-16-10-11 +.. gh-issue: 139707 +.. nonce: zR6Qtn +.. section: Library + +Improve :exc:`ModuleNotFoundError` error message when a :term:`standard +library` module is missing. + +.. + +.. date: 2025-10-15-21-42-13 +.. gh-issue: 140041 +.. nonce: _Fka2j +.. section: Library + +Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are +present. + +.. + +.. date: 2025-10-15-20-47-04 +.. gh-issue: 140120 +.. nonce: 3gffZq +.. section: Library + +Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. +Discovered by ``@ashm-dev`` using AddressSanitizer. + +.. + +.. date: 2025-10-15-17-23-51 +.. gh-issue: 140141 +.. nonce: j2mUDB +.. section: Library + +The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised +when ``importlib.metadata.Distribution.from_name`` cannot discover a +distribution no longer includes a transient :exc:`StopIteration` exception +trace. + +Contributed by Bartosz Sławecki in :gh:`140142`. + +.. + +.. date: 2025-10-15-15-10-34 +.. gh-issue: 140166 +.. nonce: NtxRez +.. section: Library + +:mod:`mimetypes`: Per the `IANA assignment +`_, update +the MIME type for the ``.texi`` and ``.texinfo`` file formats to +``application/texinfo``, instead of ``application/x-texinfo``. + +.. + +.. date: 2025-10-15-02-26-50 +.. gh-issue: 140135 +.. nonce: 54JYfM +.. section: Library + +Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x +faster) + +.. + +.. date: 2025-10-14-20-27-06 +.. gh-issue: 76007 +.. nonce: 2NcUbo +.. section: Library + +:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python +3.20. + +.. + +.. date: 2025-10-13-11-25-41 +.. gh-issue: 136702 +.. nonce: uvLGK1 +.. section: Library + +:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to +:func:`encodings.normalize_encoding` and schedule removal of support for +Python 3.17. + +.. + +.. date: 2025-10-11-09-07-06 +.. gh-issue: 139940 +.. nonce: g54efZ +.. section: Library + +Print clearer error message when using ``pdb`` to attach to a non-existing +process. + +.. + +.. date: 2025-10-02-22-29-00 +.. gh-issue: 139462 +.. nonce: VZXUHe +.. section: Library + +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID and +exit code of the terminated process. Contributed by Jonathan Berg. + +.. + +.. date: 2025-09-30-12-52-54 +.. gh-issue: 63161 +.. nonce: mECM1A +.. section: Library + +Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments +if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 +encoding. Detect null bytes in source code. + +.. + +.. date: 2025-09-25-20-16-10 +.. gh-issue: 101828 +.. nonce: yTxJlJ +.. section: Library + +Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and +``'euc_jis_2004'`` codecs truncating null chars as they were treated as part +of multi-character sequences. + +.. + +.. date: 2025-09-23-09-46-46 +.. gh-issue: 139246 +.. nonce: pzfM-w +.. section: Library + +fix: paste zero-width in default repl width is wrong. + +.. + +.. date: 2025-09-18-21-25-41 +.. gh-issue: 83714 +.. nonce: TQjDWZ +.. section: Library + +Implement :func:`os.statx` on Linux kernel versions 4.11 and later with +glibc versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor +Stinner. + +.. + +.. date: 2025-09-15-21-03-11 +.. gh-issue: 138891 +.. nonce: oZFdtR +.. section: Library + +Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is +called on a function annotated with a :pep:`646` ``star_expression`` + +.. + +.. date: 2025-09-13-12-19-17 +.. gh-issue: 138859 +.. nonce: PxjIoN +.. section: Library + +Fix generic type parameterization raising a :exc:`TypeError` when omitting a +:class:`ParamSpec` that has a default which is not a list of types. + +.. + +.. date: 2025-09-12-09-34-37 +.. gh-issue: 138764 +.. nonce: mokHoY +.. section: Library + +Prevent :func:`annotationlib.call_annotate_function` from calling +``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in +a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` +annotations in the case that neither their own format, nor +``VALUE_WITH_FAKE_GLOBALS`` are supported. + +.. + +.. date: 2025-09-11-15-03-37 +.. gh-issue: 138775 +.. nonce: w7rnSx +.. section: Library + +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from +a terminal so that it properly notices EOF. + +.. + +.. date: 2025-09-03-20-18-39 +.. gh-issue: 98896 +.. nonce: tjez89 +.. section: Library + +Fix a failure in multiprocessing resource_tracker when SharedMemory names +contain colons. Patch by Rani Pinchuk. + +.. + +.. date: 2025-09-03-18-26-07 +.. gh-issue: 138425 +.. nonce: cVE9Ho +.. section: Library + +Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which +rely on names defined as globals. + +.. + +.. date: 2025-08-26-08-17-56 +.. gh-issue: 138151 +.. nonce: I6CdAk +.. section: Library + +In :mod:`annotationlib`, improve evaluation of forward references to +nonlocal variables that are not yet defined when the annotations are +initially evaluated. + +.. + +.. date: 2025-08-15-20-35-30 +.. gh-issue: 69528 +.. nonce: qc-Eh_ +.. section: Library + +The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode +is now ``'wb+'`` instead of ``'rb+'``. + +.. + +.. date: 2025-08-11-04-52-18 +.. gh-issue: 137627 +.. nonce: Ku5Yi2 +.. section: Library + +Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. + +.. + +.. date: 2025-07-14-09-33-17 +.. gh-issue: 55531 +.. nonce: Gt2e12 +.. section: Library + +:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance +by implementing the function in C using the private +``_Py_normalize_encoding`` which has been modified to make lowercase +conversion optional. + +.. + +.. date: 2025-07-01-04-57-57 +.. gh-issue: 136057 +.. nonce: 4-t596 +.. section: Library + +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't +go over the line if a loop exists in the line. + +.. + +.. date: 2025-06-29-22-01-00 +.. gh-issue: 133390 +.. nonce: I1DW_3 +.. section: Library + +Support table, index, trigger, view, column, function, and schema completion +for :mod:`sqlite3`'s :ref:`command-line interface `. + +.. + +.. date: 2025-06-10-18-02-29 +.. gh-issue: 135307 +.. nonce: fXGrcK +.. section: Library + +:mod:`email`: Fix exception in ``set_content()`` when encoding text and +max_line_length is set to ``0`` or ``None`` (unlimited). + +.. + +.. date: 2025-05-10-15-10-54 +.. gh-issue: 133789 +.. nonce: I-ZlUX +.. section: Library + +Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. + +.. + +.. date: 2025-05-07-22-09-28 +.. gh-issue: 133601 +.. nonce: 9kUL3P +.. section: Library + +Remove deprecated :func:`!typing.no_type_check_decorator`. + +.. + +.. date: 2025-04-18-18-08-05 +.. gh-issue: 132686 +.. nonce: 6kV_Gs +.. section: Library + +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. + +.. + +.. date: 2025-03-12-18-57-10 +.. gh-issue: 131116 +.. nonce: uTpwXZ +.. section: Library + +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. + +.. + +.. date: 2025-03-04-17-19-26 +.. gh-issue: 130693 +.. nonce: Kv01r8 +.. section: Library + +Add support for ``-nolinestop``, and ``-strictlimits`` options to +:meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` +method for ``-all`` and ``-overlap`` options. + +.. + +.. date: 2024-08-08-12-39-36 +.. gh-issue: 122255 +.. nonce: J_gU8Y +.. section: Library + +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). + +.. + +.. date: 2024-06-26-16-16-43 +.. gh-issue: 121011 +.. nonce: qW54eh +.. section: Library + +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. + +.. + +.. date: 2024-05-28-17-14-30 +.. gh-issue: 119668 +.. nonce: RrIGpn +.. section: Library + +Publicly expose and document :class:`importlib.machinery.NamespacePath`. + +.. + +.. date: 2023-03-21-10-59-40 +.. gh-issue: 102431 +.. nonce: eUDnf4 +.. section: Library + +Clarify constraints for "logical" arguments in methods of +:class:`decimal.Context`. + +.. + +.. date: 2019-06-02-13-56-16 +.. gh-issue: 81313 +.. nonce: axawSH +.. section: Library + +Add the :mod:`math.integer` module (:pep:`791`). + +.. + +.. date: 2025-11-15-01-21-00 +.. gh-issue: 141579 +.. nonce: aB7cD9 +.. section: Core and Builtins + +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. + +.. + +.. date: 2025-11-14-16-25-15 +.. gh-issue: 114203 +.. nonce: n3tlQO +.. section: Core and Builtins + +Skip locking if object is already locked by two-mutex critical section. + +.. + +.. date: 2025-11-14-00-19-45 +.. gh-issue: 141528 +.. nonce: VWdax1 +.. section: Core and Builtins + +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of +the private ``_interpreters.destroy`` function when warning about remaining +subinterpreters. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-11-13-40-45 +.. gh-issue: 141367 +.. nonce: I5KY7F +.. section: Core and Builtins + +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. + +.. + +.. date: 2025-11-10-23-07-06 +.. gh-issue: 141312 +.. nonce: H-58GB +.. section: Core and Builtins + +Fix the assertion failure in the ``__setstate__`` method of the range +iterator when a non-integer argument is passed. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-05-19-50-37 +.. gh-issue: 140643 +.. nonce: QCEOqG +.. section: Core and Builtins + +Add support for ```` and ```` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. + +.. + +.. date: 2025-11-04-12-18-06 +.. gh-issue: 140942 +.. nonce: GYns6n +.. section: Core and Builtins + +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. + +.. + +.. date: 2025-11-04-04-57-24 +.. gh-issue: 140479 +.. nonce: lwQ2v2 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 21 at build time. + +.. + +.. date: 2025-11-03-17-21-38 +.. gh-issue: 140939 +.. nonce: FVboAw +.. section: Core and Builtins + +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with +the ``%*b`` format with a large width that results in a :exc:`MemoryError`. + +.. + +.. date: 2025-11-02-15-28-33 +.. gh-issue: 140260 +.. nonce: JNzlGz +.. section: Core and Builtins + +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. + +.. + +.. date: 2025-11-02-12-47-38 +.. gh-issue: 140530 +.. nonce: S934bp +.. section: Core and Builtins + +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-10-31-14-03-42 +.. gh-issue: 90344 +.. nonce: gvZigO +.. section: Core and Builtins + +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline +decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was +being called once. + +.. + +.. date: 2025-10-29-20-59-10 +.. gh-issue: 140373 +.. nonce: -uoaPP +.. section: Core and Builtins + +Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-29-11-31-59 +.. gh-issue: 140729 +.. nonce: t9JsNt +.. section: Core and Builtins + +Fix pickling error in the sampling profiler when using +``concurrent.futures.ProcessPoolExecutor`` script can not be properly +pickled and executed in worker processes. + +.. + +.. date: 2025-10-25-21-31-43 +.. gh-issue: 131527 +.. nonce: V-JVNP +.. section: Core and Builtins + +Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` +mode. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-17-36-46 +.. gh-issue: 140576 +.. nonce: kj0SCY +.. section: Core and Builtins + +Fixed crash in :func:`tokenize.generate_tokens` in case of specific +incorrect input. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-07-25-52 +.. gh-issue: 140544 +.. nonce: lwjtQe +.. section: Core and Builtins + +Speed up accessing interpreter state by caching it in a thread local +variable. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-24-20-42-33 +.. gh-issue: 140551 +.. nonce: -9swrl +.. section: Core and Builtins + +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov and Inada Naoki. + +.. + +.. date: 2025-10-24-20-16-42 +.. gh-issue: 140517 +.. nonce: cqun-K +.. section: Core and Builtins + +Fixed a reference leak when iterating over the result of :func:`map` with +``strict=True`` when the input iterables have different lengths. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-24-14-29-12 +.. gh-issue: 133467 +.. nonce: A5d6TM +.. section: Core and Builtins + +Fix race when updating :attr:`!type.__bases__` that could allow a read of +:attr:`!type.__base__` to observe an inconsistent value on the free threaded +build. + +.. + +.. date: 2025-10-23-16-05-50 +.. gh-issue: 140471 +.. nonce: Ax_aXn +.. section: Core and Builtins + +Fix potential buffer overflow in :class:`ast.AST` node initialization when +encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. + +.. + +.. date: 2025-10-22-23-26-37 +.. gh-issue: 140443 +.. nonce: wT5i1A +.. section: Core and Builtins + +The logarithm functions (such as :func:`math.log10` and :func:`math.log`) +may now produce slightly different results for extremely large integers that +cannot be converted to floats without overflow. These results are generally +more accurate, with reduced worst-case error and a tighter overall error +distribution. + +.. + +.. date: 2025-10-22-17-22-22 +.. gh-issue: 140431 +.. nonce: m8D_A- +.. section: Core and Builtins + +Fix a crash in Python's :term:`garbage collector ` due +to partially initialized :term:`coroutine` objects when coroutine origin +tracking depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). + +.. + +.. date: 2025-10-22-12-48-05 +.. gh-issue: 140476 +.. nonce: F3-d1P +.. section: Core and Builtins + +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +` build. + +.. + +.. date: 2025-10-22-11-30-16 +.. gh-issue: 135904 +.. nonce: 3WE5oW +.. section: Core and Builtins + +Add special labels to the assembly created during stencil creation to +support relocations that the native object file format does not support. +Specifically, 19 bit branches for AArch64 in Mach-O object files. + +.. + +.. date: 2025-10-21-09-20-03 +.. gh-issue: 140398 +.. nonce: SoABwJ +.. section: Core and Builtins + +Fix memory leaks in :mod:`readline` functions +:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, +:func:`~readline.write_history_file`, and +:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. + +.. + +.. date: 2025-10-21-06-51-50 +.. gh-issue: 140406 +.. nonce: 0gJs8M +.. section: Core and Builtins + +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. + +.. + +.. date: 2025-10-20-11-24-36 +.. gh-issue: 140358 +.. nonce: UQuKdV +.. section: Core and Builtins + +Restore elapsed time and unreachable object count in GC debug output. These +were inadvertently removed during a refactor of ``gc.c``. The debug log now +again reports elapsed collection time and the number of unreachable objects. +Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-19-10-32-28 +.. gh-issue: 136895 +.. nonce: HfsEh0 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 20 at build time. + +.. + +.. date: 2025-10-18-21-50-44 +.. gh-issue: 139109 +.. nonce: 9QQOzN +.. section: Core and Builtins + +A new tracing frontend for the JIT compiler has been implemented. Patch by +Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. + +.. + +.. date: 2025-10-18-21-29-45 +.. gh-issue: 140306 +.. nonce: xS5CcS +.. section: Core and Builtins + +Fix memory leaks in cross-interpreter channel operations and shared +namespace handling. + +.. + +.. date: 2025-10-18-19-52-20 +.. gh-issue: 116738 +.. nonce: NLJW0L +.. section: Core and Builtins + +Make _suggestions module thread-safe on the :term:`free threaded ` build. + +.. + +.. date: 2025-10-18-18-08-36 +.. gh-issue: 140301 +.. nonce: m-2HxC +.. section: Core and Builtins + +Fix memory leak of ``PyConfig`` in subinterpreters. + +.. + +.. date: 2025-10-17-20-23-19 +.. gh-issue: 140257 +.. nonce: 8Txmem +.. section: Core and Builtins + +Fix data race between interpreter_clear() and take_gil() on eval_breaker +during finalization with daemon threads. + +.. + +.. date: 2025-10-17-18-03-12 +.. gh-issue: 139951 +.. nonce: IdwM2O +.. section: Core and Builtins + +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. + +.. + +.. date: 2025-10-17-14-38-10 +.. gh-issue: 140253 +.. nonce: gCqFaL +.. section: Core and Builtins + +Wrong placement of a double-star pattern inside a mapping pattern now throws +a specialized syntax error. Contributed by Bartosz Sławecki in :gh:`140253`. + +.. + +.. date: 2025-10-16-21-47-00 +.. gh-issue: 140104 +.. nonce: A8SQIm +.. section: Core and Builtins + +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. + +.. + +.. date: 2025-10-15-17-12-32 +.. gh-issue: 140149 +.. nonce: cy1m3d +.. section: Core and Builtins + +Speed up parsing bytes literals concatenation by using PyBytesWriter API and +a single memory allocation (about 3x faster). + +.. + +.. date: 2025-10-15-00-21-40 +.. gh-issue: 140061 +.. nonce: J0XeDV +.. section: Core and Builtins + +Fixing the checking of whether an object is uniquely referenced to ensure +free-threaded compatibility. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-14-20-18-31 +.. gh-issue: 140080 +.. nonce: 8ROjxW +.. section: Core and Builtins + +Fix hang during finalization when attempting to call :mod:`atexit` handlers +under no memory. + +.. + +.. date: 2025-10-14-18-24-16 +.. gh-issue: 139871 +.. nonce: SWtuUz +.. section: Core and Builtins + +Update :class:`bytearray` to use a :class:`bytes` under the hood as its +buffer and add :func:`bytearray.take_bytes` to take it out. + +.. + +.. date: 2025-10-14-17-07-37 +.. gh-issue: 140067 +.. nonce: ID2gOm +.. section: Core and Builtins + +Fix memory leak in sub-interpreter creation. + +.. + +.. date: 2025-10-13-13-54-19 +.. gh-issue: 139914 +.. nonce: M-y_3E +.. section: Core and Builtins + +Restore support for HP PA-RISC, which has an upwards-growing stack. + +.. + +.. date: 2025-10-12-01-12-12 +.. gh-issue: 139817 +.. nonce: PAn-8Z +.. section: Core and Builtins + +Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. Patch +by Mikhail Efimov. + +.. + +.. date: 2025-10-06-14-19-47 +.. gh-issue: 135801 +.. nonce: OhxEZS +.. section: Core and Builtins + +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify the +module name. It is needed to unambiguous :ref:`filter ` +syntax warnings by module name. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb2 +.. section: Core and Builtins + +:func:`ast.parse` no longer emits syntax warnings for +``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they +are only emitted during compilation. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb +.. section: Core and Builtins + +Fix swallowing some syntax warnings in different modules if they +accidentally have the same message and are emitted from the same line. Fix +duplicated warnings in the ``finally`` block. + +.. + +.. date: 2025-10-03-17-51-43 +.. gh-issue: 139475 +.. nonce: _684ED +.. section: Core and Builtins + +Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use +the same pattern of refcounting for stackrefs as in production build. + +.. + +.. date: 2025-09-23-21-01-12 +.. gh-issue: 139269 +.. nonce: 1rIaxy +.. section: Core and Builtins + +Fix undefined behavior when using unaligned store in JIT's ``patch_*`` +functions. + +.. + +.. date: 2025-09-15-13-06-11 +.. gh-issue: 138944 +.. nonce: PeCgLb +.. section: Core and Builtins + +Fix :exc:`SyntaxError` message when invalid syntax appears on the same line +as a valid ``import ... as ...`` or ``from ... import ... as ...`` +statement. Patch by Brian Schubert. + +.. + +.. date: 2025-09-13-01-23-25 +.. gh-issue: 138857 +.. nonce: YQ5gdc +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for ``case`` keyword placed outside +:keyword:`match` body. + +.. + +.. date: 2025-07-29-17-51-14 +.. gh-issue: 131253 +.. nonce: GpRjWy +.. section: Core and Builtins + +Support the ``--enable-pystats`` build option for the free-threaded build. + +.. + +.. date: 2025-07-08-00-41-46 +.. gh-issue: 136327 +.. nonce: 7AiTb_ +.. section: Core and Builtins + +Errors when calling functions with invalid values after ``*`` and ``**`` now +do not include the function name. Patch by Ilia Solin. + +.. + +.. date: 2025-06-24-13-12-58 +.. gh-issue: 134786 +.. nonce: MF0VVk +.. section: Core and Builtins + +If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and +:c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` are used, then +:c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. + +.. + +.. date: 2025-11-10-11-26-26 +.. gh-issue: 141341 +.. nonce: OsO6-y +.. section: C API + +On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name +conflicts. Patch by Victor Stinner. + +.. + +.. date: 2025-11-08-10-51-50 +.. gh-issue: 116146 +.. nonce: pCmx6L +.. section: C API + +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. + +.. + +.. date: 2025-11-06-06-28-14 +.. gh-issue: 141042 +.. nonce: brOioJ +.. section: C API + +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-11-05-05-45-49 +.. gh-issue: 141004 +.. nonce: N9Ooh9 +.. section: C API + +:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. + +.. + +.. date: 2025-11-05-04-38-16 +.. gh-issue: 141004 +.. nonce: rJL43P +.. section: C API + +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. + +.. + +.. date: 2025-10-26-16-45-28 +.. gh-issue: 140556 +.. nonce: s__Dae +.. section: C API + +:pep:`793`: Add a new entry point for C extension modules, +``PyModExport_``. + +.. + +.. date: 2025-10-26-16-45-06 +.. gh-issue: 140487 +.. nonce: fGOqss +.. section: C API + +Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: +don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-10-15-15-59-59 +.. gh-issue: 140153 +.. nonce: BO7sH4 +.. section: C API + +Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by +Victor Stinner. + +.. + +.. date: 2025-10-06-22-17-47 +.. gh-issue: 139653 +.. nonce: 6-1MOd +.. section: C API + +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. + +.. + +.. date: 2025-10-31-13-20-16 +.. gh-issue: 140454 +.. nonce: gF6dCe +.. section: Build + +When building the JIT, match the jit_stencils filename expectations in +Makefile with the generator script. This avoid needless JIT recompilation +during ``make install``. + +.. + +.. date: 2025-10-29-12-30-38 +.. gh-issue: 140768 +.. nonce: ITYrzw +.. section: Build + +Warn when the WASI SDK version doesn't match what's supported. + +.. + +.. date: 2025-10-25-08-07-06 +.. gh-issue: 140513 +.. nonce: 6OhLTs +.. section: Build + +Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled +but either ``preserve_none`` or ``musttail`` is not supported. + +.. + +.. date: 2025-10-22-12-44-07 +.. gh-issue: 140475 +.. nonce: OhzQbR +.. section: Build + +Support WASI SDK 25. + +.. + +.. date: 2025-10-17-11-33-45 +.. gh-issue: 140239 +.. nonce: _k-GgW +.. section: Build + +Check ``statx`` availability only on Linux (including Android). + +.. + +.. date: 2025-10-16-11-30-53 +.. gh-issue: 140189 +.. nonce: YCrUyt +.. section: Build + +iOS builds were added to CI. + +.. + +.. date: 2025-08-10-22-28-06 +.. gh-issue: 137618 +.. nonce: FdNvIE +.. section: Build + +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. Patch by Adam +Turner. diff --git a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst deleted file mode 100644 index 0b56c4c8f68..00000000000 --- a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst +++ /dev/null @@ -1,2 +0,0 @@ -``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. -Patch by Adam Turner. diff --git a/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst b/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst deleted file mode 100644 index a1b81659242..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst +++ /dev/null @@ -1 +0,0 @@ -iOS builds were added to CI. diff --git a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst deleted file mode 100644 index 713f022c994..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst +++ /dev/null @@ -1 +0,0 @@ -Check ``statx`` availability only on Linux (including Android). diff --git a/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst b/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst deleted file mode 100644 index b4139024761..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst +++ /dev/null @@ -1 +0,0 @@ -Support WASI SDK 25. diff --git a/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst b/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst deleted file mode 100644 index 1035ebf8d78..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled but -either ``preserve_none`` or ``musttail`` is not supported. diff --git a/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst b/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst deleted file mode 100644 index 0009f83cd20..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst +++ /dev/null @@ -1 +0,0 @@ -Warn when the WASI SDK version doesn't match what's supported. diff --git a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst deleted file mode 100644 index 4bb132ce01e..00000000000 --- a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst +++ /dev/null @@ -1,3 +0,0 @@ -When building the JIT, match the jit_stencils filename expectations in -Makefile with the generator script. This avoid needless JIT recompilation -during ``make install``. diff --git a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst deleted file mode 100644 index cd3d5262fa0..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and -:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the -stack protection base address and stack protection size of a Python thread -state. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst b/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst deleted file mode 100644 index 502c48b6842..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst deleted file mode 100644 index 16b0d9d4084..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: -don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst deleted file mode 100644 index 61da60903ee..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst +++ /dev/null @@ -1,2 +0,0 @@ -:pep:`793`: Add a new entry point for C extension modules, -``PyModExport_``. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst deleted file mode 100644 index a054f8eda6f..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst +++ /dev/null @@ -1 +0,0 @@ -The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst deleted file mode 100644 index 5f3ccd62016..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst +++ /dev/null @@ -1 +0,0 @@ -:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst deleted file mode 100644 index 22a1aa1f405..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while -conversion to a narrower precision floating-point format --- the remaining -after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst deleted file mode 100644 index be8043e26dd..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a -module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst deleted file mode 100644 index 460923b4d62..00000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name -conflicts. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst deleted file mode 100644 index 664e4d2db38..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst +++ /dev/null @@ -1,2 +0,0 @@ -If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` -are used, then :c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst deleted file mode 100644 index 3798e956c95..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Errors when calling functions with invalid values after ``*`` and ``**`` now do not -include the function name. Patch by Ilia Solin. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst deleted file mode 100644 index 2826fad2330..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst +++ /dev/null @@ -1 +0,0 @@ -Support the ``--enable-pystats`` build option for the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst deleted file mode 100644 index 93510a9ceaf..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`SyntaxError` message for ``case`` keyword placed outside -:keyword:`match` body. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst deleted file mode 100644 index 248585e2eba..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :exc:`SyntaxError` message when invalid syntax appears on the same line -as a valid ``import ... as ...`` or ``from ... import ... as ...`` -statement. Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst deleted file mode 100644 index e36be529d2a..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behavior when using unaligned store in JIT's ``patch_*`` functions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst deleted file mode 100644 index f4d50b7d020..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use -the same pattern of refcounting for stackrefs as in production build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst deleted file mode 100644 index 396e40f0e13..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix swallowing some syntax warnings in different modules if they -accidentally have the same message and are emitted from the same line. -Fix duplicated warnings in the ``finally`` block. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst deleted file mode 100644 index b147b430ccc..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`ast.parse` no longer emits syntax warnings for -``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they are -only emitted during compilation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst deleted file mode 100644 index 96226a7c525..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst +++ /dev/null @@ -1,6 +0,0 @@ -Many functions related to compiling or parsing Python code, such as -:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and -:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify -the module name. -It is needed to unambiguous :ref:`filter ` syntax warnings -by module name. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst deleted file mode 100644 index b205d21edfe..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst +++ /dev/null @@ -1,2 +0,0 @@ -Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. -Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst deleted file mode 100644 index 7529108d5d4..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst +++ /dev/null @@ -1 +0,0 @@ -Restore support for HP PA-RISC, which has an upwards-growing stack. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst deleted file mode 100644 index 3c5a828101d..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in sub-interpreter creation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst deleted file mode 100644 index d4b8578afe3..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :class:`bytearray` to use a :class:`bytes` under the hood as its buffer -and add :func:`bytearray.take_bytes` to take it out. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst deleted file mode 100644 index 0ddcea57f9d..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang during finalization when attempting to call :mod:`atexit` handlers under no memory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst deleted file mode 100644 index 7c3924195eb..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixing the checking of whether an object is uniquely referenced to ensure -free-threaded compatibility. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst deleted file mode 100644 index e98e28802cf..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up parsing bytes literals concatenation by using PyBytesWriter API and -a single memory allocation (about 3x faster). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst deleted file mode 100644 index 1c18cbc9ad0..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported -by Daniel Diniz. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst deleted file mode 100644 index 955dcac2e01..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Wrong placement of a double-star pattern inside a mapping pattern now throws a specialized syntax error. -Contributed by Bartosz Sławecki in :gh:`140253`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst deleted file mode 100644 index e03996188a7..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fixes a regression in GC performance for a growing heap composed mostly of -small tuples. - -* Counts number of actually tracked objects, instead of trackable objects. - This ensures that untracking tuples has the desired effect of reducing GC overhead. -* Does not track most untrackable tuples during creation. - This prevents large numbers of small tuples causing excessive GCs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst deleted file mode 100644 index 50f7e0e48ae..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix data race between interpreter_clear() and take_gil() on eval_breaker -during finalization with daemon threads. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst deleted file mode 100644 index 8b1c81c04ec..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak of ``PyConfig`` in subinterpreters. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst deleted file mode 100644 index bf323b870bc..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make _suggestions module thread-safe on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst deleted file mode 100644 index 2178c496063..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leaks in cross-interpreter channel operations and shared -namespace handling. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst deleted file mode 100644 index 40b9d19ee42..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst +++ /dev/null @@ -1 +0,0 @@ -A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst deleted file mode 100644 index fffc264a865..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst +++ /dev/null @@ -1 +0,0 @@ -Update JIT compilation to use LLVM 20 at build time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst deleted file mode 100644 index 739228f7e36..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore elapsed time and unreachable object count in GC debug output. These -were inadvertently removed during a refactor of ``gc.c``. The debug log now -again reports elapsed collection time and the number of unreachable objects. -Contributed by Pål Grønås Drange. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst deleted file mode 100644 index 3506ba42581..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak when an object's :meth:`~object.__hash__` method returns an -object that isn't an :class:`int`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst deleted file mode 100644 index 481dac7f26d..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix memory leaks in :mod:`readline` functions -:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, -:func:`~readline.write_history_file`, and -:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst deleted file mode 100644 index b52a57dba4a..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add special labels to the assembly created during stencil creation to -support relocations that the native object file format does not support. -Specifically, 19 bit branches for AArch64 in Mach-O object files. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst deleted file mode 100644 index a24033208c5..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded -` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst deleted file mode 100644 index 3d62d210f1f..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in Python's :term:`garbage collector ` due to -partially initialized :term:`coroutine` objects when coroutine origin tracking -depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst deleted file mode 100644 index a1fff8fef7e..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst +++ /dev/null @@ -1,5 +0,0 @@ -The logarithm functions (such as :func:`math.log10` and :func:`math.log`) may now produce -slightly different results for extremely large integers that cannot be -converted to floats without overflow. These results are generally more -accurate, with reduced worst-case error and a tighter overall error -distribution. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst deleted file mode 100644 index afa9326fff3..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential buffer overflow in :class:`ast.AST` node initialization when -encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst deleted file mode 100644 index f69786866e9..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst +++ /dev/null @@ -1 +0,0 @@ -Fix race when updating :attr:`!type.__bases__` that could allow a read of :attr:`!type.__base__` to observe an inconsistent value on the free threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst deleted file mode 100644 index 15aaea8ab02..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a reference leak when iterating over the result of :func:`map` -with ``strict=True`` when the input iterables have different lengths. -Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst deleted file mode 100644 index 8fd9b46c0ae..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup -stage. Patch by Mikhail Efimov and Inada Naoki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst deleted file mode 100644 index 51d2b229ee5..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up accessing interpreter state by caching it in a thread local variable. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst deleted file mode 100644 index 2c27525d9f7..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash in :func:`tokenize.generate_tokens` in case of -specific incorrect input. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst deleted file mode 100644 index 9969ea058a3..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` -mode. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst deleted file mode 100644 index 6725547667f..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` -script can not be properly pickled and executed in worker processes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst deleted file mode 100644 index c9a97037920..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by -Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst deleted file mode 100644 index b1d05354f65..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst +++ /dev/null @@ -1 +0,0 @@ -Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was being called once. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst deleted file mode 100644 index e3af493893a..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt -Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst deleted file mode 100644 index 96bf9b51e48..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`struct` data race in endian table initialization with -subinterpreters. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst deleted file mode 100644 index a2921761f75..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with the -``%*b`` format with a large width that results in a :exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst deleted file mode 100644 index 0a615ed1311..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst +++ /dev/null @@ -1 +0,0 @@ -Update JIT compilation to use LLVM 21 at build time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst deleted file mode 100644 index 20cfeca1e71..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of -``application/node``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst deleted file mode 100644 index e1202dd1a17..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support for ```` and ```` frames to -:mod:`!profiling.sampling` output to denote active garbage collection and -calls to native code. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst deleted file mode 100644 index fdb136cef3f..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the assertion failure in the ``__setstate__`` method of the range iterator -when a non-integer argument is passed. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst deleted file mode 100644 index cb830fcd9e1..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list -subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst deleted file mode 100644 index a51aa495228..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of the -private ``_interpreters.destroy`` function when warning about remaining subinterpreters. -Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst deleted file mode 100644 index 883f9333cae..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst +++ /dev/null @@ -1 +0,0 @@ -Skip locking if object is already locked by two-mutex critical section. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst deleted file mode 100644 index 8ab9979c399..00000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`sys.activate_stack_trampoline` to properly support the -``perf_jit`` backend. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst b/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst deleted file mode 100644 index 2291c938222..00000000000 --- a/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst +++ /dev/null @@ -1 +0,0 @@ -Add the :mod:`math.integer` module (:pep:`791`). diff --git a/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst b/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst deleted file mode 100644 index e82ddb6e101..00000000000 --- a/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Clarify constraints for "logical" arguments in methods of -:class:`decimal.Context`. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst b/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst deleted file mode 100644 index 87cdf8d89d5..00000000000 --- a/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst +++ /dev/null @@ -1 +0,0 @@ -Publicly expose and document :class:`importlib.machinery.NamespacePath`. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst deleted file mode 100644 index aee7fe2bcb5..00000000000 --- a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`math.log` now supports arbitrary large integer-like arguments in the -same way as arbitrary large integer arguments. diff --git a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst deleted file mode 100644 index 63e71c19f8b..00000000000 --- a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst +++ /dev/null @@ -1,4 +0,0 @@ -In the :mod:`linecache` module and in the Python implementation of the -:mod:`warnings` module, a ``DeprecationWarning`` is issued when -``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C -implementation of the :mod:`!warnings` module). diff --git a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst deleted file mode 100644 index b175ab7cad4..00000000000 --- a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for ``-nolinestop``, and ``-strictlimits`` options to :meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` method for ``-all`` and ``-overlap`` options. diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst deleted file mode 100644 index f5e60ab6e8c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`inspect.getdoc` now correctly returns an inherited docstring on -:class:`~functools.cached_property` objects if none is given in a subclass. diff --git a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst deleted file mode 100644 index d0c8e2d705c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add parameters *inherit_class_doc* and *fallback_to_class_doc* for -:func:`inspect.getdoc`. diff --git a/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst b/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst deleted file mode 100644 index 62f40aee7aa..00000000000 --- a/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated :func:`!typing.no_type_check_decorator`. diff --git a/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst b/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst deleted file mode 100644 index d2a4f7f42c3..00000000000 --- a/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst +++ /dev/null @@ -1 +0,0 @@ -Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst b/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst deleted file mode 100644 index 47e1feb5cbf..00000000000 --- a/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email`: Fix exception in ``set_content()`` when encoding text -and max_line_length is set to ``0`` or ``None`` (unlimited). diff --git a/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst b/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst deleted file mode 100644 index c57f802d4c8..00000000000 --- a/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support table, index, trigger, view, column, function, and schema completion -for :mod:`sqlite3`'s :ref:`command-line interface `. diff --git a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst deleted file mode 100644 index e237a0e98cc..00000000000 --- a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't go over the line if a loop exists in the line. diff --git a/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst b/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst deleted file mode 100644 index 70e39a4f2c1..00000000000 --- a/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance -by implementing the function in C using the private -``_Py_normalize_encoding`` which has been modified to make lowercase -conversion optional. diff --git a/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst b/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst deleted file mode 100644 index 855070ed6f4..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. diff --git a/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst b/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst deleted file mode 100644 index b18781e0dce..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode is -now ``'wb+'`` instead of ``'rb+'``. diff --git a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst b/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst deleted file mode 100644 index de29f536afc..00000000000 --- a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`annotationlib`, improve evaluation of forward references to -nonlocal variables that are not yet defined when the annotations are -initially evaluated. diff --git a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst b/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst deleted file mode 100644 index 328e5988cb0..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which rely -on names defined as globals. diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst deleted file mode 100644 index 6831499c0af..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. -Patch by Rani Pinchuk. diff --git a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst deleted file mode 100644 index 455c1a9925a..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use of ``python -m`` with :mod:`base64` has been fixed to detect input from a -terminal so that it properly notices EOF. diff --git a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst deleted file mode 100644 index 85ebef8ff11..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Prevent :func:`annotationlib.call_annotate_function` from calling ``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in a fake globals namespace with empty globals. - -Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` annotations in the case that neither their own format, nor ``VALUE_WITH_FAKE_GLOBALS`` are supported. diff --git a/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst b/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst deleted file mode 100644 index a5d4dd042fc..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst +++ /dev/null @@ -1 +0,0 @@ -Fix generic type parameterization raising a :exc:`TypeError` when omitting a :class:`ParamSpec` that has a default which is not a list of types. diff --git a/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst b/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst deleted file mode 100644 index f7ecb05d20c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is -called on a function annotated with a :pep:`646` ``star_expression`` diff --git a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst deleted file mode 100644 index 3653eb9a114..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement :func:`os.statx` on Linux kernel versions 4.11 and later with glibc -versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst b/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst deleted file mode 100644 index a816bda5cfe..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst +++ /dev/null @@ -1 +0,0 @@ -fix: paste zero-width in default repl width is wrong. diff --git a/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst b/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst deleted file mode 100644 index 1d100180c07..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and -``'euc_jis_2004'`` codecs truncating null chars -as they were treated as part of multi-character sequences. diff --git a/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst b/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst deleted file mode 100644 index 3daed20d099..00000000000 --- a/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments -if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 -encoding. Detect null bytes in source code. diff --git a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst deleted file mode 100644 index 390a6124386..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst +++ /dev/null @@ -1,3 +0,0 @@ -When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` -terminates abruptly, the resulting traceback will now tell you the PID -and exit code of the terminated process. Contributed by Jonathan Berg. diff --git a/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst b/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst deleted file mode 100644 index 2501135e657..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst +++ /dev/null @@ -1 +0,0 @@ -Print clearer error message when using ``pdb`` to attach to a non-existing process. diff --git a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst deleted file mode 100644 index 88303f017f5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to -:func:`encodings.normalize_encoding` and schedule removal of support for -Python 3.17. diff --git a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst b/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst deleted file mode 100644 index 567fb5ef904..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python -3.20. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst b/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst deleted file mode 100644 index 8d5a76af909..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x -faster) diff --git a/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst b/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst deleted file mode 100644 index c140db9dcd5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`mimetypes`: Per the `IANA assignment `_, update the MIME type for the ``.texi`` and ``.texinfo`` file formats to ``application/texinfo``, instead of ``application/x-texinfo``. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst b/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst deleted file mode 100644 index 2edadbc3e38..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst +++ /dev/null @@ -1,5 +0,0 @@ -The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised when -``importlib.metadata.Distribution.from_name`` cannot discover a -distribution no longer includes a transient :exc:`StopIteration` exception trace. - -Contributed by Bartosz Sławecki in :gh:`140142`. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst b/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst deleted file mode 100644 index 9eefe140520..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. -Discovered by ``@ashm-dev`` using AddressSanitizer. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst b/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst deleted file mode 100644 index 243ff39311c..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst +++ /dev/null @@ -1 +0,0 @@ -Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are present. diff --git a/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst b/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst deleted file mode 100644 index c5460aae8b3..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`ModuleNotFoundError` error message when a :term:`standard library` -module is missing. diff --git a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst b/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst deleted file mode 100644 index d680312d582..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst +++ /dev/null @@ -1,6 +0,0 @@ -Improve filtering by module in :func:`warnings.warn_explicit` if no *module* -argument is passed. It now tests the module regular expression in the -warnings filter not only against the filename with ``.py`` stripped, but -also against module names constructed starting from different parent -directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, -``.pyw`` stripped). diff --git a/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst b/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst deleted file mode 100644 index 5563d077171..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst +++ /dev/null @@ -1,5 +0,0 @@ -Calendar's HTML formatting now accepts year and month as options. -Previously, running ``python -m calendar -t html 2025 10`` would result in an -error message. It now generates an HTML document displaying the calendar for -the specified month. -Contributed by Pål Grønås Drange. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst b/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst deleted file mode 100644 index cb08e02429b..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst +++ /dev/null @@ -1 +0,0 @@ -Colorize the default import statement ``import asyncio`` in asyncio REPL. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst b/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst deleted file mode 100644 index 8767b1bb483..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and -:func:`~unicodedata.isxidcontinue` functions to check whether a character can -start or continue a `Unicode Standard Annex #31 `_ identifier. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst b/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst deleted file mode 100644 index 666a45055f5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst b/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst deleted file mode 100644 index be56b2ca6a1..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst +++ /dev/null @@ -1 +0,0 @@ -Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst b/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst deleted file mode 100644 index 6a91fc41b0a..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`decimal`: Deprecate ``__version__`` and replace with -:data:`decimal.SPEC_VERSION`. diff --git a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst b/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst deleted file mode 100644 index 16d5b2a8bf0..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix regression in Python 3.14.0 where using the ``|`` operator on a -:class:`typing.Union` object combined with an object that is not a type -would raise an error. diff --git a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst deleted file mode 100644 index 4ff55b41dea..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by replacing the annotations that were in-line in the generated source code with ``__annotate__`` functions attached to the methods. diff --git a/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst b/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst deleted file mode 100644 index db7f92e136d..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change the default of ``suggest_on_error`` to ``True`` in -``argparse.ArgumentParser``. diff --git a/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst b/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst deleted file mode 100644 index aca4e68b1e5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak in :class:`array.array` when creating arrays from an empty -:class:`str` and the ``u`` type code. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst b/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst deleted file mode 100644 index e12f789e674..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`ast.unparse` now generates full source code when handling -:class:`ast.Interpolation` nodes that do not have a specified source. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst b/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst deleted file mode 100644 index 1f511c3b9d0..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst +++ /dev/null @@ -1 +0,0 @@ -Improve error message when trying to iterate a Tk widget, image or font. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst b/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst deleted file mode 100644 index ef7a90bc37e..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the -*extra* argument. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst deleted file mode 100644 index cc33217c9f5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned by -:meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst deleted file mode 100644 index 612ad82dc64..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with -:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom -element declaration handler. Patch by Sebastian Pipping. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst deleted file mode 100644 index 72666bb8224..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` -when the iterator is not explicitly closed and was opened with a filename. -This helps developers identify and fix resource leaks. Patch by Osama -Abdelkader. diff --git a/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst b/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst deleted file mode 100644 index 9675a5d427a..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute -when loading an extension module packaged as Apple Framework. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst deleted file mode 100644 index 2ae153a6480..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where closing :class:`io.BufferedWriter` could crash if -the closed attribute raised an exception on access or could not be -converted to a boolean. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst b/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst deleted file mode 100644 index b1ba9b26ad5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst b/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst deleted file mode 100644 index 299e9f04df7..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Display thread name in :mod:`faulthandler` on Windows. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst b/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst deleted file mode 100644 index 802183673cf..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix arguments checking for the :meth:`!functools.partial.__setstate__` that -may lead to internal state corruption and crash. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst deleted file mode 100644 index 4c68d4cd94b..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst +++ /dev/null @@ -1 +0,0 @@ -Error and warning keywords in ``argparse.ArgumentParser`` messages are now colorized when color output is enabled, fixing a visual inconsistency in which they remained plain text while other output was colorized. diff --git a/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst b/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst deleted file mode 100644 index b3b692bae62..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid making unnecessary filesystem calls for frozen modules in :mod:`linecache` when the global module cache is not present. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst b/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst deleted file mode 100644 index 9fa8c561a03..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`profiling.sampling.sample` incorrectly handling a -:exc:`FileNotFoundError` or :exc:`PermissionError`. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst b/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst deleted file mode 100644 index f6b42be1fbf..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst +++ /dev/null @@ -1 +0,0 @@ -Add :func:`os.reload_environ` to ``os.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst b/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst deleted file mode 100644 index fce8dd33757..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst +++ /dev/null @@ -1 +0,0 @@ -Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst deleted file mode 100644 index 03856f0b9b6..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst +++ /dev/null @@ -1 +0,0 @@ -Initialize all Pdb's instance variables in ``__init__``, remove some hasattr/getattr diff --git a/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst b/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst deleted file mode 100644 index 7ccbfc3cb95..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Convert ``_json`` module to use Argument Clinic. -Patched by Yoonho Hann. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst deleted file mode 100644 index e14af7d9708..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst deleted file mode 100644 index 84b6195c926..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`urllib.request`, when opening a FTP URL fails because a data -connection cannot be made, the control connection's socket is now closed to -avoid a :exc:`ResourceWarning`. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst b/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst deleted file mode 100644 index 090f39c6e25..00000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst +++ /dev/null @@ -1 +0,0 @@ -The internal class ``mailbox._ProxyFile`` is no longer a parameterized generic. diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst deleted file mode 100644 index 875d15f2f89..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst +++ /dev/null @@ -1,2 +0,0 @@ -Now :class:`!winreg.HKEYType` objects are compared by their underlying Windows -registry handle value instead of their object identity. diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst deleted file mode 100644 index a48162de76b..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst +++ /dev/null @@ -1 +0,0 @@ -Bump the version of pip bundled in ensurepip to version 25.3 diff --git a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst deleted file mode 100644 index e1505764064..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support of non-:term:`descriptor` callables in -:func:`functools.singledispatchmethod`. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst deleted file mode 100644 index 46582f7fcf4..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing`: fix off-by-one error when checking the length -of a temporary socket file path. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst deleted file mode 100644 index 2f7500d2955..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or -:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt -Tran. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst deleted file mode 100644 index 18c4d3836ef..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`faulthandler` now detects if a frame or a code object is invalid or -freed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst deleted file mode 100644 index d36debec3ed..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor the :mod:`pdb` parsing issue so positional arguments can pass through intuitively. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst deleted file mode 100644 index 493b740261e..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst +++ /dev/null @@ -1,2 +0,0 @@ -The undocumented :class:`!re.Scanner` class now forbids regular expressions containing capturing groups in its lexicon patterns. Patterns using capturing groups could -previously lead to crashes with segmentation fault. Use non-capturing groups (?:...) instead. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst deleted file mode 100644 index 9a31af9c110..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst +++ /dev/null @@ -1 +0,0 @@ -Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst deleted file mode 100644 index dfa582bdbc8..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :meth:`annotationlib.ForwardRef.evaluate` returning -:class:`~annotationlib.ForwardRef` objects which don't update with new -globals. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst deleted file mode 100644 index e776515a9fb..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when -``strict=False``) ``.jpg`` to their correct IANA mime type. diff --git a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst deleted file mode 100644 index f59ccfb33e7..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a thread safety issue with :func:`base64.b85decode`. Contributed by Benel Tayar. diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst deleted file mode 100644 index 3e4fd1a5897..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, -to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` -when a file is in "w" mode rather than :exc:`OSError` diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst deleted file mode 100644 index 62073280e32..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst deleted file mode 100644 index bb425ce5df3..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior -arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst deleted file mode 100644 index 37acaabfa3e..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst +++ /dev/null @@ -1 +0,0 @@ -Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files with standalone carriage return (``\r``) line endings. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst deleted file mode 100644 index 32f4e39f6d5..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst +++ /dev/null @@ -1,5 +0,0 @@ -The :mod:`os.fork` and related forking APIs will no longer warn in the -common case where Linux or macOS platform APIs return the number of threads -in a process and find the answer to be 1 even when a -:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a -thread. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst deleted file mode 100644 index 8436cd2415d..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Updated Tcl threading configuration in :mod:`_tkinter` to assume that -threads are always available in Tcl 9 and later. diff --git a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst deleted file mode 100644 index bd3044002a2..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now raise a -:exc:`ValueError` when the input contains an infinity or a NaN. diff --git a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst deleted file mode 100644 index 328bfe067ad..00000000000 --- a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`ipaddress`: ensure that the methods -:meth:`IPv4Network.hosts() ` and -:meth:`IPv6Network.hosts() ` always return an -iterator. diff --git a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst deleted file mode 100644 index 1d152bb5318..00000000000 --- a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst +++ /dev/null @@ -1 +0,0 @@ -Fix quadratic complexity in :func:`os.path.expandvars`. diff --git a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst deleted file mode 100644 index 940a3ad5a72..00000000000 --- a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email.message`: ensure linear complexity for legacy HTTP parameters -parsing. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst b/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst deleted file mode 100644 index c30c9439a76..00000000000 --- a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", -"noembed" and "noframes", and optionally RAWTEXT element "noscript" in -:class:`html.parser.HTMLParser`. diff --git a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst deleted file mode 100644 index f87fb1113ca..00000000000 --- a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst +++ /dev/null @@ -1 +0,0 @@ -Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` raises an exception diff --git a/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst b/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst deleted file mode 100644 index 70e70218254..00000000000 --- a/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color -enabled so that :mod:`unittest` which is run by it with redirected output will -output in color. diff --git a/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst b/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst deleted file mode 100644 index 20747ad7f11..00000000000 --- a/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst +++ /dev/null @@ -1 +0,0 @@ -Preserve and restore the state of ``stty echo`` as part of the test environment. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst b/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst deleted file mode 100644 index 9f52d0163ab..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``Tools/tz/zdump.py`` script. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst b/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst deleted file mode 100644 index 0dc589c3986..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``Tools/scripts/checkpip.py`` script. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst b/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst deleted file mode 100644 index 9efbf0162dd..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst +++ /dev/null @@ -1,2 +0,0 @@ -The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment -variable to iOS apps being tested. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst deleted file mode 100644 index 073c070413f..00000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst +++ /dev/null @@ -1 +0,0 @@ -The iOS testbed now correctly handles test arguments that contain spaces. diff --git a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst deleted file mode 100644 index 6f25b867566..00000000000 --- a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled liblzma to version 5.8.1. diff --git a/README.rst b/README.rst index a228aafb09c..bc1c1df2069 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.15.0 alpha 1 +This is Python version 3.15.0 alpha 2 ===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push From a52c39e2608557a710784d5876150578d2ae5183 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 18 Nov 2025 15:14:16 +0000 Subject: [PATCH 233/417] gh-135953: Refactor test_sampling_profiler into multiple files (#141689) --- .../test_profiling/test_sampling_profiler.py | 3360 ----------------- .../test_sampling_profiler/__init__.py | 9 + .../test_sampling_profiler/helpers.py | 101 + .../test_sampling_profiler/mocks.py | 38 + .../test_sampling_profiler/test_advanced.py | 264 ++ .../test_sampling_profiler/test_cli.py | 664 ++++ .../test_sampling_profiler/test_collectors.py | 896 +++++ .../test_integration.py | 804 ++++ .../test_sampling_profiler/test_modes.py | 514 +++ .../test_sampling_profiler/test_profiler.py | 656 ++++ Makefile.pre.in | 1 + 11 files changed, 3947 insertions(+), 3360 deletions(-) delete mode 100644 Lib/test/test_profiling/test_sampling_profiler.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/__init__.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/helpers.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/mocks.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_advanced.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_cli.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_collectors.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_integration.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_modes.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_profiler.py diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py deleted file mode 100644 index c2cc2ddd48a..00000000000 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ /dev/null @@ -1,3360 +0,0 @@ -"""Tests for the sampling profiler (profiling.sampling).""" - -import contextlib -import io -import json -import marshal -import os -import shutil -import socket -import subprocess -import sys -import tempfile -import unittest -from collections import namedtuple -from unittest import mock - -from profiling.sampling.pstats_collector import PstatsCollector -from profiling.sampling.stack_collector import ( - CollapsedStackCollector, - FlamegraphCollector, -) -from profiling.sampling.gecko_collector import GeckoCollector - -from test.support.os_helper import unlink -from test.support import ( - force_not_colorized_test_class, - SHORT_TIMEOUT, - script_helper, - os_helper, - SuppressCrashReport, -) -from test.support.socket_helper import find_unused_port -from test.support import requires_subprocess, is_emscripten -from test.support import captured_stdout, captured_stderr - -PROCESS_VM_READV_SUPPORTED = False - -try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - import _remote_debugging -except ImportError: - raise unittest.SkipTest( - "Test only runs when _remote_debugging is available" - ) -else: - import profiling.sampling - from profiling.sampling.sample import SampleProfiler - - - -class MockFrameInfo: - """Mock FrameInfo for testing since the real one isn't accessible.""" - - def __init__(self, filename, lineno, funcname): - self.filename = filename - self.lineno = lineno - self.funcname = funcname - - def __repr__(self): - return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" - - -class MockThreadInfo: - """Mock ThreadInfo for testing since the real one isn't accessible.""" - - def __init__(self, thread_id, frame_info, status=0): # Default to THREAD_STATE_RUNNING (0) - self.thread_id = thread_id - self.frame_info = frame_info - self.status = status - - def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" - - -class MockInterpreterInfo: - """Mock InterpreterInfo for testing since the real one isn't accessible.""" - - def __init__(self, interpreter_id, threads): - self.interpreter_id = interpreter_id - self.threads = threads - - def __repr__(self): - return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" - - -skip_if_not_supported = unittest.skipIf( - ( - sys.platform != "darwin" - and sys.platform != "linux" - and sys.platform != "win32" - ), - "Test only runs on Linux, Windows and MacOS", -) - -SubprocessInfo = namedtuple('SubprocessInfo', ['process', 'socket']) - - -@contextlib.contextmanager -def test_subprocess(script): - # Find an unused port for socket communication - port = find_unused_port() - - # Inject socket connection code at the beginning of the script - socket_code = f''' -import socket -_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -_test_sock.connect(('localhost', {port})) -_test_sock.sendall(b"ready") -''' - - # Combine socket code with user script - full_script = socket_code + script - - # Create server socket to wait for process to be ready - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - - proc = subprocess.Popen( - [sys.executable, "-c", full_script], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - client_socket = None - try: - # Wait for process to connect and send ready signal - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - if response != b"ready": - raise RuntimeError(f"Unexpected response from subprocess: {response}") - - yield SubprocessInfo(proc, client_socket) - finally: - if client_socket is not None: - client_socket.close() - if proc.poll() is None: - proc.kill() - proc.wait() - - -def close_and_unlink(file): - file.close() - unlink(file.name) - - -class TestSampleProfilerComponents(unittest.TestCase): - """Unit tests for individual profiler components.""" - - def test_mock_frame_info_with_empty_and_unicode_values(self): - """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" - # Test with empty strings - frame = MockFrameInfo("", 0, "") - self.assertEqual(frame.filename, "") - self.assertEqual(frame.lineno, 0) - self.assertEqual(frame.funcname, "") - self.assertIn("filename=''", repr(frame)) - - # Test with unicode characters - frame = MockFrameInfo("文件.py", 42, "函数名") - self.assertEqual(frame.filename, "文件.py") - self.assertEqual(frame.funcname, "函数名") - - # Test with very long names - long_filename = "x" * 1000 + ".py" - long_funcname = "func_" + "x" * 1000 - frame = MockFrameInfo(long_filename, 999999, long_funcname) - self.assertEqual(frame.filename, long_filename) - self.assertEqual(frame.lineno, 999999) - self.assertEqual(frame.funcname, long_funcname) - - def test_pstats_collector_with_extreme_intervals_and_empty_data(self): - """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" - # Test with zero interval - collector = PstatsCollector(sample_interval_usec=0) - self.assertEqual(collector.sample_interval_usec, 0) - - # Test with very large interval - collector = PstatsCollector(sample_interval_usec=1000000000) - self.assertEqual(collector.sample_interval_usec, 1000000000) - - # Test collecting empty frames list - collector = PstatsCollector(sample_interval_usec=1000) - collector.collect([]) - self.assertEqual(len(collector.result), 0) - - # Test collecting frames with None thread id - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])])] - collector.collect(test_frames) - # Should still process the frames - self.assertEqual(len(collector.result), 1) - - # Test collecting duplicate frames in same sample - test_frames = [ - MockInterpreterInfo( - 0, # interpreter_id - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 10, "func1"), # Duplicate - ], - )] - ) - ] - collector = PstatsCollector(sample_interval_usec=1000) - collector.collect(test_frames) - # Should count both occurrences - self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 - ) - - def test_pstats_collector_single_frame_stacks(self): - """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Test with exactly one frame (should trigger the <= 1 condition) - single_frame = [MockInterpreterInfo(0, [MockThreadInfo(1, [MockFrameInfo("single.py", 10, "single_func")])])] - collector.collect(single_frame) - - # Should record the single frame with inline call - self.assertEqual(len(collector.result), 1) - single_key = ("single.py", 10, "single_func") - self.assertIn(single_key, collector.result) - self.assertEqual(collector.result[single_key]["direct_calls"], 1) - self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) - - # Test with empty frames (should also trigger <= 1 condition) - empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] - collector.collect(empty_frames) - - # Should not add any new entries - self.assertEqual( - len(collector.result), 1 - ) # Still just the single frame - - # Test mixed single and multi-frame stacks - mixed_frames = [ - MockInterpreterInfo( - 0, - [ - MockThreadInfo( - 1, - [MockFrameInfo("single2.py", 20, "single_func2")], - ), # Single frame - MockThreadInfo( - 2, - [ # Multi-frame stack - MockFrameInfo("multi.py", 30, "multi_func1"), - MockFrameInfo("multi.py", 40, "multi_func2"), - ], - ), - ] - ), - ] - collector.collect(mixed_frames) - - # Should have recorded all functions - self.assertEqual( - len(collector.result), 4 - ) # single + single2 + multi1 + multi2 - - # Verify single frame handling - single2_key = ("single2.py", 20, "single_func2") - self.assertIn(single2_key, collector.result) - self.assertEqual(collector.result[single2_key]["direct_calls"], 1) - self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) - - # Verify multi-frame handling still works - multi1_key = ("multi.py", 30, "multi_func1") - multi2_key = ("multi.py", 40, "multi_func2") - self.assertIn(multi1_key, collector.result) - self.assertIn(multi2_key, collector.result) - self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) - self.assertEqual( - collector.result[multi2_key]["cumulative_calls"], 1 - ) # Called from multi1 - - def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): - """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" - collector = CollapsedStackCollector() - - # Test with empty frames - collector.collect([]) - self.assertEqual(len(collector.stack_counter), 0) - - # Test with single frame stack - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func")])])] - collector.collect(test_frames) - self.assertEqual(len(collector.stack_counter), 1) - ((path, thread_id), count), = collector.stack_counter.items() - self.assertEqual(path, (("file.py", 10, "func"),)) - self.assertEqual(thread_id, 1) - self.assertEqual(count, 1) - - # Test with very deep stack - deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] - collector = CollapsedStackCollector() - collector.collect(test_frames) - # One aggregated path with 100 frames (reversed) - ((path_tuple, thread_id),), = (collector.stack_counter.keys(),) - self.assertEqual(len(path_tuple), 100) - self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) - self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) - self.assertEqual(thread_id, 1) - - def test_pstats_collector_basic(self): - """Test basic PstatsCollector functionality.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Test empty state - self.assertEqual(len(collector.result), 0) - self.assertEqual(len(collector.stats), 0) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 20, "func2"), - ], - )] - ) - ] - collector.collect(test_frames) - - # Should have recorded calls for both functions - self.assertEqual(len(collector.result), 2) - self.assertIn(("file.py", 10, "func1"), collector.result) - self.assertIn(("file.py", 20, "func2"), collector.result) - - # Top-level function should have direct call - self.assertEqual( - collector.result[("file.py", 10, "func1")]["direct_calls"], 1 - ) - self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 - ) - - # Calling function should have cumulative call but no direct calls - self.assertEqual( - collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 - ) - self.assertEqual( - collector.result[("file.py", 20, "func2")]["direct_calls"], 0 - ) - - def test_pstats_collector_create_stats(self): - """Test PstatsCollector stats creation.""" - collector = PstatsCollector( - sample_interval_usec=1000000 - ) # 1 second intervals - - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 20, "func2"), - ], - )] - ) - ] - collector.collect(test_frames) - collector.collect(test_frames) # Collect twice - - collector.create_stats() - - # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) - func1_stats = collector.stats[("file.py", 10, "func1")] - self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) - self.assertEqual(func1_stats[1], 2) # cumulative_calls - self.assertEqual( - func1_stats[2], 2.0 - ) # tt (total time - 2 samples * 1 sec) - self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) - - func2_stats = collector.stats[("file.py", 20, "func2")] - self.assertEqual( - func2_stats[0], 0 - ) # direct_calls (never top of stack) - self.assertEqual( - func2_stats[1], 2 - ) # cumulative_calls (appears in stack) - self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) - self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) - - def test_collapsed_stack_collector_basic(self): - collector = CollapsedStackCollector() - - # Test empty state - self.assertEqual(len(collector.stack_counter), 0) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] - collector.collect(test_frames) - - # Should store one reversed path - self.assertEqual(len(collector.stack_counter), 1) - ((path, thread_id), count), = collector.stack_counter.items() - expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) - self.assertEqual(path, expected_tree) - self.assertEqual(thread_id, 1) - self.assertEqual(count, 1) - - def test_collapsed_stack_collector_export(self): - collapsed_out = tempfile.NamedTemporaryFile(delete=False) - self.addCleanup(close_and_unlink, collapsed_out) - - collector = CollapsedStackCollector() - - test_frames1 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] - test_frames2 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] # Same stack - test_frames3 = [MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])])] - - collector.collect(test_frames1) - collector.collect(test_frames2) - collector.collect(test_frames3) - - with (captured_stdout(), captured_stderr()): - collector.export(collapsed_out.name) - # Check file contents - with open(collapsed_out.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertEqual(len(lines), 2) # Two unique stacks - - # Check collapsed format: tid:X;file:func:line;file:func:line count - stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" - stack2_expected = "tid:1;other.py:other_func:5 1" - - self.assertIn(stack1_expected, lines) - self.assertIn(stack2_expected, lines) - - def test_flamegraph_collector_basic(self): - """Test basic FlamegraphCollector functionality.""" - collector = FlamegraphCollector() - - # Empty collector should produce 'No Data' - data = collector._convert_to_flamegraph_format() - # With string table, name is now an index - resolve it using the strings array - strings = data.get("strings", []) - name_index = data.get("name", 0) - resolved_name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index) - self.assertIn(resolved_name, ("No Data", "No significant data")) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] - collector.collect(test_frames) - - # Convert and verify structure: func2 -> func1 with counts = 1 - data = collector._convert_to_flamegraph_format() - # Expect promotion: root is the single child (func2), with func1 as its only child - strings = data.get("strings", []) - name_index = data.get("name", 0) - name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index) - self.assertIsInstance(name, str) - self.assertTrue(name.startswith("Program Root: ")) - self.assertIn("func2 (file.py:20)", name) # formatted name - children = data.get("children", []) - self.assertEqual(len(children), 1) - child = children[0] - child_name_index = child.get("name", 0) - child_name = strings[child_name_index] if isinstance(child_name_index, int) and 0 <= child_name_index < len(strings) else str(child_name_index) - self.assertIn("func1 (file.py:10)", child_name) # formatted name - self.assertEqual(child["value"], 1) - - def test_flamegraph_collector_export(self): - """Test flamegraph HTML export functionality.""" - flamegraph_out = tempfile.NamedTemporaryFile( - suffix=".html", delete=False - ) - self.addCleanup(close_and_unlink, flamegraph_out) - - collector = FlamegraphCollector() - - # Create some test data (use Interpreter/Thread objects like runtime) - test_frames1 = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] - test_frames2 = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] # Same stack - test_frames3 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])]) - ] - - collector.collect(test_frames1) - collector.collect(test_frames2) - collector.collect(test_frames3) - - # Export flamegraph - with (captured_stdout(), captured_stderr()): - collector.export(flamegraph_out.name) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(flamegraph_out.name)) - self.assertGreater(os.path.getsize(flamegraph_out.name), 0) - - # Check file contains HTML content - with open(flamegraph_out.name, "r", encoding="utf-8") as f: - content = f.read() - - # Should be valid HTML - self.assertIn("", content.lower()) - self.assertIn(" 0) - self.assertGreater(mock_collector.collect.call_count, 0) - self.assertLessEqual(mock_collector.collect.call_count, 3) - - def test_sample_profiler_missed_samples_warning(self): - """Test that the profiler warns about missed samples when sampling is too slow.""" - from profiling.sampling.sample import SampleProfiler - - mock_unwinder = mock.MagicMock() - mock_unwinder.get_stack_trace.return_value = [ - ( - 1, - [ - mock.MagicMock( - filename="test.py", lineno=10, funcname="test_func" - ) - ], - ) - ] - - with mock.patch( - "_remote_debugging.RemoteUnwinder" - ) as mock_unwinder_class: - mock_unwinder_class.return_value = mock_unwinder - - # Use very short interval that we'll miss - profiler = SampleProfiler( - pid=12345, sample_interval_usec=1000, all_threads=False - ) # 1ms interval - - mock_collector = mock.MagicMock() - - # Simulate slow sampling where we miss many samples - times = [ - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - ] # Extra time points to avoid StopIteration - - with mock.patch("time.perf_counter", side_effect=times): - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - profiler.sample(mock_collector, duration_sec=0.5) - - result = output.getvalue() - - # Should warn about missed samples - self.assertIn("Warning: missed", result) - self.assertIn("samples from the expected total", result) - - -@force_not_colorized_test_class -class TestPrintSampledStats(unittest.TestCase): - """Test the print_sampled_stats function.""" - - def setUp(self): - """Set up test data.""" - # Mock stats data - self.mock_stats = mock.MagicMock() - self.mock_stats.stats = { - ("file1.py", 10, "func1"): ( - 100, - 100, - 0.5, - 0.5, - {}, - ), # cc, nc, tt, ct, callers - ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), - ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), - ("file4.py", 40, "func4"): ( - 10, - 10, - 0.001, - 0.001, - {}, - ), # millisecond range - ("file5.py", 50, "func5"): ( - 5, - 5, - 0.000001, - 0.000002, - {}, - ), # microsecond range - } - - def test_print_sampled_stats_basic(self): - """Test basic print_sampled_stats functionality.""" - from profiling.sampling.sample import print_sampled_stats - - # Capture output - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(self.mock_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Check header is present - self.assertIn("Profile Stats:", result) - self.assertIn("nsamples", result) - self.assertIn("tottime", result) - self.assertIn("cumtime", result) - - # Check functions are present - self.assertIn("func1", result) - self.assertIn("func2", result) - self.assertIn("func3", result) - - def test_print_sampled_stats_sorting(self): - """Test different sorting options.""" - from profiling.sampling.sample import print_sampled_stats - - # Test sort by calls - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=0, sample_interval_usec=100 - ) - - result = output.getvalue() - lines = result.strip().split("\n") - - # Find the data lines (skip header) - data_lines = [l for l in lines if "file" in l and ".py" in l] - # func3 should be first (200 calls) - self.assertIn("func3", data_lines[0]) - - # Test sort by time - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=1, sample_interval_usec=100 - ) - - result = output.getvalue() - lines = result.strip().split("\n") - - data_lines = [l for l in lines if "file" in l and ".py" in l] - # func3 should be first (1.5s time) - self.assertIn("func3", data_lines[0]) - - def test_print_sampled_stats_limit(self): - """Test limiting output rows.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, limit=2, sample_interval_usec=100 - ) - - result = output.getvalue() - - # Count function entries in the main stats section (not in summary) - lines = result.split("\n") - # Find where the main stats section ends (before summary) - main_section_lines = [] - for line in lines: - if "Summary of Interesting Functions:" in line: - break - main_section_lines.append(line) - - # Count function entries only in main section - func_count = sum( - 1 - for line in main_section_lines - if "func" in line and ".py" in line - ) - self.assertEqual(func_count, 2) - - def test_print_sampled_stats_time_units(self): - """Test proper time unit selection.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(self.mock_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should use seconds for the header since max time is > 1s - self.assertIn("tottime (s)", result) - self.assertIn("cumtime (s)", result) - - # Test with only microsecond-range times - micro_stats = mock.MagicMock() - micro_stats.stats = { - ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(micro_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should use microseconds - self.assertIn("tottime (μs)", result) - self.assertIn("cumtime (μs)", result) - - def test_print_sampled_stats_summary(self): - """Test summary section generation.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Check summary sections are present - self.assertIn("Summary of Interesting Functions:", result) - self.assertIn( - "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", - result, - ) - self.assertIn( - "Functions with Highest Call Frequency (Indirect Calls):", result - ) - self.assertIn( - "Functions with Highest Call Magnification (Cumulative/Direct):", - result, - ) - - def test_print_sampled_stats_no_summary(self): - """Test disabling summary output.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, - show_summary=False, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Summary should not be present - self.assertNotIn("Summary of Interesting Functions:", result) - - def test_print_sampled_stats_empty_stats(self): - """Test with empty stats.""" - from profiling.sampling.sample import print_sampled_stats - - empty_stats = mock.MagicMock() - empty_stats.stats = {} - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(empty_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should still print header - self.assertIn("Profile Stats:", result) - - def test_print_sampled_stats_sample_percentage_sorting(self): - """Test sample percentage sorting options.""" - from profiling.sampling.sample import print_sampled_stats - - # Add a function with high sample percentage (more direct calls than func3's 200) - self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( - 300, # direct calls (higher than func3's 200) - 300, # cumulative calls - 1.0, # total time - 1.0, # cumulative time - {}, - ) - - # Test sort by sample percentage - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=3, sample_interval_usec=100 - ) # sample percentage - - result = output.getvalue() - lines = result.strip().split("\n") - - data_lines = [l for l in lines if ".py" in l and "func" in l] - # expensive_func should be first (highest sample percentage) - self.assertIn("expensive_func", data_lines[0]) - - def test_print_sampled_stats_with_recursive_calls(self): - """Test print_sampled_stats with recursive calls where nc != cc.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with recursive calls (nc != cc) - recursive_stats = mock.MagicMock() - recursive_stats.stats = { - # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function - ("recursive.py", 10, "factorial"): ( - 5, # direct_calls - 10, # cumulative_calls (appears more times in stack due to recursion) - 0.5, - 0.6, - {}, - ), - ("normal.py", 20, "normal_func"): ( - 3, # direct_calls - 3, # cumulative_calls (same as direct for non-recursive) - 0.2, - 0.2, - {}, - ), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(recursive_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should display recursive calls as "5/10" format - self.assertIn("5/10", result) # nc/cc format for recursive calls - self.assertIn("3", result) # just nc for non-recursive calls - self.assertIn("factorial", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_zero_call_counts(self): - """Test print_sampled_stats with zero call counts to trigger division protection.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with zero call counts - zero_stats = mock.MagicMock() - zero_stats.stats = { - ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls - ("file.py", 20, "normal_func"): ( - 5, - 5, - 0.1, - 0.1, - {}, - ), # Normal function - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(zero_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should handle zero call counts gracefully - self.assertIn("zero_calls", result) - self.assertIn("zero_calls", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_sort_by_name(self): - """Test sort by function name option.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=-1, sample_interval_usec=100 - ) # sort by name - - result = output.getvalue() - lines = result.strip().split("\n") - - # Find the data lines (skip header and summary) - # Data lines start with whitespace and numbers, and contain filename:lineno(function) - data_lines = [] - for line in lines: - # Skip header lines and summary sections - if ( - line.startswith(" ") - and "(" in line - and ")" in line - and not line.startswith( - " 1." - ) # Skip summary lines that start with times - and not line.startswith( - " 0." - ) # Skip summary lines that start with times - and not "per call" in line # Skip summary lines - and not "calls" in line # Skip summary lines - and not "total time" in line # Skip summary lines - and not "cumulative time" in line - ): # Skip summary lines - data_lines.append(line) - - # Extract just the function names for comparison - func_names = [] - import re - - for line in data_lines: - # Function name is between the last ( and ), accounting for ANSI color codes - match = re.search(r"\(([^)]+)\)$", line) - if match: - func_name = match.group(1) - # Remove ANSI color codes - func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) - func_names.append(func_name) - - # Verify we extracted function names and they are sorted - self.assertGreater( - len(func_names), 0, "Should have extracted some function names" - ) - self.assertEqual( - func_names, - sorted(func_names), - f"Function names {func_names} should be sorted alphabetically", - ) - - def test_print_sampled_stats_with_zero_time_functions(self): - """Test summary sections with functions that have zero time.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with zero-time functions - zero_time_stats = mock.MagicMock() - zero_time_stats.stats = { - ("file1.py", 10, "zero_time_func"): ( - 5, - 5, - 0.0, - 0.0, - {}, - ), # Zero time - ("file2.py", 20, "normal_func"): ( - 3, - 3, - 0.1, - 0.1, - {}, - ), # Normal time - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - zero_time_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Should handle zero-time functions gracefully in summary - self.assertIn("Summary of Interesting Functions:", result) - self.assertIn("zero_time_func", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_malformed_qualified_names(self): - """Test summary generation with function names that don't contain colons.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with function names that would create malformed qualified names - malformed_stats = mock.MagicMock() - malformed_stats.stats = { - # Function name without clear module separation - ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), - ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), - ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - malformed_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Should handle malformed names gracefully in summary aggregation - self.assertIn("Summary of Interesting Functions:", result) - # All function names should appear somewhere in the output - self.assertIn("func", result) - self.assertIn("empty_filename_func", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_recursive_call_stats_creation(self): - """Test create_stats with recursive call data to trigger total_rec_calls branch.""" - collector = PstatsCollector(sample_interval_usec=1000000) # 1 second - - # Simulate recursive function data where total_rec_calls would be set - # We need to manually manipulate the collector result to test this branch - collector.result = { - ("recursive.py", 10, "factorial"): { - "total_rec_calls": 3, # Non-zero recursive calls - "direct_calls": 5, - "cumulative_calls": 10, - }, - ("normal.py", 20, "normal_func"): { - "total_rec_calls": 0, # Zero recursive calls - "direct_calls": 2, - "cumulative_calls": 5, - }, - } - - collector.create_stats() - - # Check that recursive calls are handled differently from non-recursive - factorial_stats = collector.stats[("recursive.py", 10, "factorial")] - normal_stats = collector.stats[("normal.py", 20, "normal_func")] - - # factorial should use cumulative_calls (10) as nc - self.assertEqual( - factorial_stats[1], 10 - ) # nc should be cumulative_calls - self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls - - # normal_func should use cumulative_calls as nc - self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls - self.assertEqual(normal_stats[0], 2) # cc should be direct_calls - - -@skip_if_not_supported -@unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", -) -class TestRecursiveFunctionProfiling(unittest.TestCase): - """Test profiling of recursive functions and complex call patterns.""" - - def test_recursive_function_call_counting(self): - """Test that recursive function calls are counted correctly.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate a recursive call pattern: fibonacci(5) calling itself - recursive_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # First sample: deep in recursion - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call - MockFrameInfo( - "fib.py", 10, "fibonacci" - ), # deeper recursion - MockFrameInfo("fib.py", 10, "fibonacci"), # even deeper - MockFrameInfo("main.py", 5, "main"), # main caller - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Second sample: different recursion depth - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call - MockFrameInfo("main.py", 5, "main"), # main caller - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Third sample: back to deeper recursion - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in recursive_frames: - collector.collect([frames]) - - collector.create_stats() - - # Check that recursive calls are counted properly - fib_key = ("fib.py", 10, "fibonacci") - main_key = ("main.py", 5, "main") - - self.assertIn(fib_key, collector.stats) - self.assertIn(main_key, collector.stats) - - # Fibonacci should have many calls due to recursion - fib_stats = collector.stats[fib_key] - direct_calls, cumulative_calls, tt, ct, callers = fib_stats - - # Should have recorded multiple calls (9 total appearances in samples) - self.assertEqual(cumulative_calls, 9) - self.assertGreater(tt, 0) # Should have some total time - self.assertGreater(ct, 0) # Should have some cumulative time - - # Main should have fewer calls - main_stats = collector.stats[main_key] - main_direct_calls, main_cumulative_calls = main_stats[0], main_stats[1] - self.assertEqual(main_direct_calls, 0) # Never directly executing - self.assertEqual(main_cumulative_calls, 3) # Appears in all 3 samples - - def test_nested_function_hierarchy(self): - """Test profiling of deeply nested function calls.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate a deep call hierarchy - deep_call_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("level1.py", 10, "level1_func"), - MockFrameInfo("level2.py", 20, "level2_func"), - MockFrameInfo("level3.py", 30, "level3_func"), - MockFrameInfo("level4.py", 40, "level4_func"), - MockFrameInfo("level5.py", 50, "level5_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Same hierarchy sampled again - MockFrameInfo("level1.py", 10, "level1_func"), - MockFrameInfo("level2.py", 20, "level2_func"), - MockFrameInfo("level3.py", 30, "level3_func"), - MockFrameInfo("level4.py", 40, "level4_func"), - MockFrameInfo("level5.py", 50, "level5_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in deep_call_frames: - collector.collect([frames]) - - collector.create_stats() - - # All levels should be recorded - for level in range(1, 6): - key = (f"level{level}.py", level * 10, f"level{level}_func") - self.assertIn(key, collector.stats) - - stats = collector.stats[key] - direct_calls, cumulative_calls, tt, ct, callers = stats - - # Each level should appear in stack twice (2 samples) - self.assertEqual(cumulative_calls, 2) - - # Only level1 (deepest) should have direct calls - if level == 1: - self.assertEqual(direct_calls, 2) - else: - self.assertEqual(direct_calls, 0) - - # Deeper levels should have lower cumulative time than higher levels - # (since they don't include time from functions they call) - if level == 1: # Deepest level with most time - self.assertGreater(ct, 0) - - def test_alternating_call_patterns(self): - """Test profiling with alternating call patterns.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate alternating execution paths - pattern_frames = [ - # Pattern A: path through func_a - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 10, "func_a"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern B: path through func_b - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 20, "func_b"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern A again - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 10, "func_a"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern B again - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 20, "func_b"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in pattern_frames: - collector.collect([frames]) - - collector.create_stats() - - # Check that both paths are recorded equally - func_a_key = ("module.py", 10, "func_a") - func_b_key = ("module.py", 20, "func_b") - shared_key = ("module.py", 30, "shared_func") - main_key = ("main.py", 5, "main") - - # func_a and func_b should each be directly executing twice - self.assertEqual(collector.stats[func_a_key][0], 2) # direct_calls - self.assertEqual(collector.stats[func_a_key][1], 2) # cumulative_calls - self.assertEqual(collector.stats[func_b_key][0], 2) # direct_calls - self.assertEqual(collector.stats[func_b_key][1], 2) # cumulative_calls - - # shared_func should appear in all samples (4 times) but never directly executing - self.assertEqual(collector.stats[shared_key][0], 0) # direct_calls - self.assertEqual(collector.stats[shared_key][1], 4) # cumulative_calls - - # main should appear in all samples but never directly executing - self.assertEqual(collector.stats[main_key][0], 0) # direct_calls - self.assertEqual(collector.stats[main_key][1], 4) # cumulative_calls - - def test_collapsed_stack_with_recursion(self): - """Test collapsed stack collector with recursive patterns.""" - collector = CollapsedStackCollector() - - # Recursive call pattern - recursive_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - ("factorial.py", 10, "factorial"), - ("factorial.py", 10, "factorial"), # recursive - ("factorial.py", 10, "factorial"), # deeper - ("main.py", 5, "main"), - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - ("factorial.py", 10, "factorial"), - ("factorial.py", 10, "factorial"), # different depth - ("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in recursive_frames: - collector.collect([frames]) - - # Should capture both call paths - self.assertEqual(len(collector.stack_counter), 2) - - # First path should be longer (deeper recursion) than the second - path_tuples = list(collector.stack_counter.keys()) - paths = [p[0] for p in path_tuples] # Extract just the call paths - lengths = [len(p) for p in paths] - self.assertNotEqual(lengths[0], lengths[1]) - - # Both should contain factorial calls - self.assertTrue(any(any(f[2] == "factorial" for f in p) for p in paths)) - - # Verify total occurrences via aggregation - factorial_key = ("factorial.py", 10, "factorial") - main_key = ("main.py", 5, "main") - - def total_occurrences(func): - total = 0 - for (path, thread_id), count in collector.stack_counter.items(): - total += sum(1 for f in path if f == func) * count - return total - - self.assertEqual(total_occurrences(factorial_key), 5) - self.assertEqual(total_occurrences(main_key), 2) - - -@requires_subprocess() -@skip_if_not_supported -class TestSampleProfilerIntegration(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.test_script = ''' -import time -import os - -def slow_fibonacci(n): - """Recursive fibonacci - should show up prominently in profiler.""" - if n <= 1: - return n - return slow_fibonacci(n-1) + slow_fibonacci(n-2) - -def cpu_intensive_work(): - """CPU intensive work that should show in profiler.""" - result = 0 - for i in range(10000): - result += i * i - if i % 100 == 0: - result = result % 1000000 - return result - -def medium_computation(): - """Medium complexity function.""" - result = 0 - for i in range(100): - result += i * i - return result - -def fast_loop(): - """Fast simple loop.""" - total = 0 - for i in range(50): - total += i - return total - -def nested_calls(): - """Test nested function calls.""" - def level1(): - def level2(): - return medium_computation() - return level2() - return level1() - -def main_loop(): - """Main test loop with different execution paths.""" - iteration = 0 - - while True: - iteration += 1 - - # Different execution paths - focus on CPU intensive work - if iteration % 3 == 0: - # Very CPU intensive - result = cpu_intensive_work() - elif iteration % 5 == 0: - # Expensive recursive operation - result = slow_fibonacci(12) - else: - # Medium operation - result = nested_calls() - - # No sleep - keep CPU busy - -if __name__ == "__main__": - main_loop() -''' - - def test_sampling_basic_functionality(self): - with ( - test_subprocess(self.test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2, - sample_interval_usec=1000, # 1ms - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - def test_sampling_with_pstats_export(self): - pstats_out = tempfile.NamedTemporaryFile( - suffix=".pstats", delete=False - ) - self.addCleanup(close_and_unlink, pstats_out) - - with test_subprocess(self.test_script) as subproc: - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=pstats_out.name, - sample_interval_usec=10000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(pstats_out.name)) - self.assertGreater(os.path.getsize(pstats_out.name), 0) - - # Try to load the stats file - with open(pstats_out.name, "rb") as f: - stats_data = marshal.load(f) - - # Should be a dictionary with the sampled marker - self.assertIsInstance(stats_data, dict) - self.assertIn(("__sampled__",), stats_data) - self.assertTrue(stats_data[("__sampled__",)]) - - # Should have some function data - function_entries = [ - k for k in stats_data.keys() if k != ("__sampled__",) - ] - self.assertGreater(len(function_entries), 0) - - def test_sampling_with_collapsed_export(self): - collapsed_file = tempfile.NamedTemporaryFile( - suffix=".txt", delete=False - ) - self.addCleanup(close_and_unlink, collapsed_file) - - with ( - test_subprocess(self.test_script) as subproc, - ): - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=collapsed_file.name, - output_format="collapsed", - sample_interval_usec=10000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(collapsed_file.name)) - self.assertGreater(os.path.getsize(collapsed_file.name), 0) - - # Check file format - with open(collapsed_file.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertGreater(len(lines), 0) - - # Each line should have format: stack_trace count - for line in lines: - parts = line.rsplit(" ", 1) - self.assertEqual(len(parts), 2) - - stack_trace, count_str = parts - self.assertGreater(len(stack_trace), 0) - self.assertTrue(count_str.isdigit()) - self.assertGreater(int(count_str), 0) - - # Stack trace should contain semicolon-separated entries - if ";" in stack_trace: - stack_parts = stack_trace.split(";") - for part in stack_parts: - # Each part should be file:function:line - self.assertIn(":", part) - - def test_sampling_all_threads(self): - with ( - test_subprocess(self.test_script) as subproc, - # Suppress profiler output - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - all_threads=True, - sample_interval_usec=10000, - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - # Just verify that sampling completed without error - # We're not testing output format here - - def test_sample_target_script(self): - script_file = tempfile.NamedTemporaryFile(delete=False) - script_file.write(self.test_script.encode("utf-8")) - script_file.flush() - self.addCleanup(close_and_unlink, script_file) - - test_args = ["profiling.sampling.sample", "-d", "1", script_file.name] - - with ( - mock.patch("sys.argv", test_args), - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.main() - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - def test_sample_target_module(self): - tempdir = tempfile.TemporaryDirectory(delete=False) - self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) - - module_path = os.path.join(tempdir.name, "test_module.py") - - with open(module_path, "w") as f: - f.write(self.test_script) - - test_args = ["profiling.sampling.sample", "-d", "1", "-m", "test_module"] - - with ( - mock.patch("sys.argv", test_args), - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - # Change to temp directory so subprocess can find the module - contextlib.chdir(tempdir.name), - ): - try: - profiling.sampling.sample.main() - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - -@skip_if_not_supported -@unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", -) -class TestSampleProfilerErrorHandling(unittest.TestCase): - def test_invalid_pid(self): - with self.assertRaises((OSError, RuntimeError)): - profiling.sampling.sample.sample(-1, duration_sec=1) - - def test_process_dies_during_sampling(self): - with test_subprocess("import time; time.sleep(0.5); exit()") as subproc: - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2, # Longer than process lifetime - sample_interval_usec=50000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - output = captured_output.getvalue() - - self.assertIn("Error rate", output) - - def test_invalid_output_format(self): - with self.assertRaises(ValueError): - profiling.sampling.sample.sample( - os.getpid(), - duration_sec=1, - output_format="invalid_format", - ) - - def test_invalid_output_format_with_mocked_profiler(self): - """Test invalid output format with proper mocking to avoid permission issues.""" - with mock.patch( - "profiling.sampling.sample.SampleProfiler" - ) as mock_profiler_class: - mock_profiler = mock.MagicMock() - mock_profiler_class.return_value = mock_profiler - - with self.assertRaises(ValueError) as cm: - profiling.sampling.sample.sample( - 12345, - duration_sec=1, - output_format="unknown_format", - ) - - # Should raise ValueError with the invalid format name - self.assertIn( - "Invalid output format: unknown_format", str(cm.exception) - ) - - def test_is_process_running(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: - try: - profiler = SampleProfiler(pid=subproc.process.pid, sample_interval_usec=1000, all_threads=False) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - self.assertTrue(profiler._is_process_running()) - self.assertIsNotNone(profiler.unwinder.get_stack_trace()) - subproc.process.kill() - subproc.process.wait() - self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) - - # Exit the context manager to ensure the process is terminated - self.assertFalse(profiler._is_process_running()) - self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) - - @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") - def test_esrch_signal_handling(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: - try: - unwinder = _remote_debugging.RemoteUnwinder(subproc.process.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - initial_trace = unwinder.get_stack_trace() - self.assertIsNotNone(initial_trace) - - subproc.process.kill() - - # Wait for the process to die and try to get another trace - subproc.process.wait() - - with self.assertRaises(ProcessLookupError): - unwinder.get_stack_trace() - - def test_valid_output_formats(self): - """Test that all valid output formats are accepted.""" - valid_formats = ["pstats", "collapsed", "flamegraph", "gecko"] - - tempdir = tempfile.TemporaryDirectory(delete=False) - self.addCleanup(shutil.rmtree, tempdir.name) - - - with (contextlib.chdir(tempdir.name), captured_stdout(), captured_stderr()): - for fmt in valid_formats: - try: - # This will likely fail with permissions, but the format should be valid - profiling.sampling.sample.sample( - os.getpid(), - duration_sec=0.1, - output_format=fmt, - filename=f"test_{fmt}.out", - ) - except (OSError, RuntimeError, PermissionError): - # Expected errors - we just want to test format validation - pass - - def test_script_error_treatment(self): - script_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".py") - script_file.write("open('nonexistent_file.txt')\n") - script_file.close() - self.addCleanup(os.unlink, script_file.name) - - result = subprocess.run( - [sys.executable, "-m", "profiling.sampling.sample", "-d", "1", script_file.name], - capture_output=True, - text=True, - ) - output = result.stdout + result.stderr - - if "PermissionError" in output: - self.skipTest("Insufficient permissions for remote profiling") - self.assertNotIn("Script file not found", output) - self.assertIn("No such file or directory: 'nonexistent_file.txt'", output) - - -class TestSampleProfilerCLI(unittest.TestCase): - def _setup_sync_mocks(self, mock_socket, mock_popen): - """Helper to set up socket and process mocks for coordinator tests.""" - # Mock the sync socket with context manager support - mock_sock_instance = mock.MagicMock() - mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) - - # Mock the connection with context manager support - mock_conn = mock.MagicMock() - mock_conn.recv.return_value = b"ready" - mock_conn.__enter__.return_value = mock_conn - mock_conn.__exit__.return_value = None - - # Mock accept() to return (connection, address) and support indexing - mock_accept_result = mock.MagicMock() - mock_accept_result.__getitem__.return_value = mock_conn # [0] returns the connection - mock_sock_instance.accept.return_value = mock_accept_result - - # Mock socket with context manager support - mock_sock_instance.__enter__.return_value = mock_sock_instance - mock_sock_instance.__exit__.return_value = None - mock_socket.return_value = mock_sock_instance - - # Mock the subprocess - mock_process = mock.MagicMock() - mock_process.pid = 12345 - mock_process.poll.return_value = None - mock_popen.return_value = mock_process - return mock_process - - def _verify_coordinator_command(self, mock_popen, expected_target_args): - """Helper to verify the coordinator command was called correctly.""" - args, kwargs = mock_popen.call_args - coordinator_cmd = args[0] - self.assertEqual(coordinator_cmd[0], sys.executable) - self.assertEqual(coordinator_cmd[1], "-m") - self.assertEqual(coordinator_cmd[2], "profiling.sampling._sync_coordinator") - self.assertEqual(coordinator_cmd[3], "12345") # port - # cwd is coordinator_cmd[4] - self.assertEqual(coordinator_cmd[5:], expected_target_args) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_argument_parsing(self): - test_args = ["profiling.sampling.sample", "-m", "mymodule"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) - mock_sample.assert_called_once_with( - 12345, - sort=2, # default sort (sort_value from args.sort) - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_with_arguments(self): - test_args = ["profiling.sampling.sample", "-m", "mymodule", "arg1", "arg2", "--flag"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag")) - mock_sample.assert_called_once_with( - 12345, - sort=2, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_argument_parsing(self): - test_args = ["profiling.sampling.sample", "myscript.py"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("myscript.py",)) - mock_sample.assert_called_once_with( - 12345, - sort=2, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_with_arguments(self): - test_args = ["profiling.sampling.sample", "myscript.py", "arg1", "arg2", "--flag"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - # Use the helper to set up mocks consistently - mock_process = self._setup_sync_mocks(mock_socket, mock_popen) - # Override specific behavior for this test - mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] - - profiling.sampling.sample.main() - - # Verify the coordinator command was called - args, kwargs = mock_popen.call_args - coordinator_cmd = args[0] - self.assertEqual(coordinator_cmd[0], sys.executable) - self.assertEqual(coordinator_cmd[1], "-m") - self.assertEqual(coordinator_cmd[2], "profiling.sampling._sync_coordinator") - self.assertEqual(coordinator_cmd[3], "12345") # port - # cwd is coordinator_cmd[4] - self.assertEqual(coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag")) - - def test_cli_mutually_exclusive_pid_module(self): - test_args = ["profiling.sampling.sample", "-p", "12345", "-m", "mymodule"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("not allowed with argument", error_msg) - - def test_cli_mutually_exclusive_pid_script(self): - test_args = ["profiling.sampling.sample", "-p", "12345", "myscript.py"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("only one target type can be specified", error_msg) - - def test_cli_no_target_specified(self): - test_args = ["profiling.sampling.sample", "-d", "5"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("one of the arguments", error_msg) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_with_profiler_options(self): - test_args = [ - "profiling.sampling.sample", "-i", "1000", "-d", "30", "-a", - "--sort-tottime", "-l", "20", "-m", "mymodule", - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) - mock_sample.assert_called_once_with( - 12345, - sort=1, # sort-tottime - sample_interval_usec=1000, - duration_sec=30, - filename=None, - all_threads=True, - limit=20, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_with_profiler_options(self): - """Test script with various profiler options.""" - test_args = [ - "profiling.sampling.sample", "-i", "2000", "-d", "60", - "--collapsed", "-o", "output.txt", - "myscript.py", "scriptarg", - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("myscript.py", "scriptarg")) - # Verify profiler options were passed correctly - mock_sample.assert_called_once_with( - 12345, - sort=2, # default sort - sample_interval_usec=2000, - duration_sec=60, - filename="output.txt", - all_threads=False, - limit=15, - show_summary=True, - output_format="collapsed", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - def test_cli_empty_module_name(self): - test_args = ["profiling.sampling.sample", "-m"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("argument -m/--module: expected one argument", error_msg) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_long_module_option(self): - test_args = ["profiling.sampling.sample", "--module", "mymodule", "arg1"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1")) - - def test_cli_complex_script_arguments(self): - test_args = [ - "profiling.sampling.sample", "script.py", - "--input", "file.txt", "-v", "--output=/tmp/out", "positional" - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("profiling.sampling.sample._run_with_sync") as mock_run_with_sync, - ): - mock_process = mock.MagicMock() - mock_process.pid = 12345 - mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] - mock_process.poll.return_value = None - mock_run_with_sync.return_value = mock_process - - profiling.sampling.sample.main() - - mock_run_with_sync.assert_called_once_with(( - sys.executable, "script.py", - "--input", "file.txt", "-v", "--output=/tmp/out", "positional", - )) - - def test_cli_collapsed_format_validation(self): - """Test that CLI properly validates incompatible options with collapsed format.""" - test_cases = [ - # Test sort options are invalid with collapsed - ( - ["profiling.sampling.sample", "--collapsed", "--sort-nsamples", "-p", "12345"], - "sort", - ), - ( - ["profiling.sampling.sample", "--collapsed", "--sort-tottime", "-p", "12345"], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-cumtime", - "-p", - "12345", - ], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-sample-pct", - "-p", - "12345", - ], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-cumul-pct", - "-p", - "12345", - ], - "sort", - ), - ( - ["profiling.sampling.sample", "--collapsed", "--sort-name", "-p", "12345"], - "sort", - ), - # Test limit option is invalid with collapsed - (["profiling.sampling.sample", "--collapsed", "-l", "20", "-p", "12345"], "limit"), - ( - ["profiling.sampling.sample", "--collapsed", "--limit", "20", "-p", "12345"], - "limit", - ), - # Test no-summary option is invalid with collapsed - ( - ["profiling.sampling.sample", "--collapsed", "--no-summary", "-p", "12345"], - "summary", - ), - ] - - for test_args, expected_error_keyword in test_cases: - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error code - error_msg = mock_stderr.getvalue() - self.assertIn("error:", error_msg) - self.assertIn("--pstats format", error_msg) - - def test_cli_default_collapsed_filename(self): - """Test that collapsed format gets a default filename when not specified.""" - test_args = ["profiling.sampling.sample", "--collapsed", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - # Check that filename was set to default collapsed format - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["output_format"], "collapsed") - self.assertEqual(call_args["filename"], "collapsed.12345.txt") - - def test_cli_custom_output_filenames(self): - """Test custom output filenames for both formats.""" - test_cases = [ - ( - ["profiling.sampling.sample", "--pstats", "-o", "custom.pstats", "-p", "12345"], - "custom.pstats", - "pstats", - ), - ( - ["profiling.sampling.sample", "--collapsed", "-o", "custom.txt", "-p", "12345"], - "custom.txt", - "collapsed", - ), - ] - - for test_args, expected_filename, expected_format in test_cases: - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["filename"], expected_filename) - self.assertEqual(call_args["output_format"], expected_format) - - def test_cli_missing_required_arguments(self): - """Test that CLI requires PID argument.""" - with ( - mock.patch("sys.argv", ["profiling.sampling.sample"]), - mock.patch("sys.stderr", io.StringIO()), - ): - with self.assertRaises(SystemExit): - profiling.sampling.sample.main() - - def test_cli_mutually_exclusive_format_options(self): - """Test that pstats and collapsed options are mutually exclusive.""" - with ( - mock.patch( - "sys.argv", - ["profiling.sampling.sample", "--pstats", "--collapsed", "-p", "12345"], - ), - mock.patch("sys.stderr", io.StringIO()), - ): - with self.assertRaises(SystemExit): - profiling.sampling.sample.main() - - def test_argument_parsing_basic(self): - test_args = ["profiling.sampling.sample", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once_with( - 12345, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - sort=2, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - def test_sort_options(self): - sort_options = [ - ("--sort-nsamples", 0), - ("--sort-tottime", 1), - ("--sort-cumtime", 2), - ("--sort-sample-pct", 3), - ("--sort-cumul-pct", 4), - ("--sort-name", -1), - ] - - for option, expected_sort_value in sort_options: - test_args = ["profiling.sampling.sample", option, "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual( - call_args["sort"], - expected_sort_value, - ) - mock_sample.reset_mock() - - -class TestCpuModeFiltering(unittest.TestCase): - """Test CPU mode filtering functionality (--mode=cpu).""" - - def test_mode_validation(self): - """Test that CLI validates mode choices correctly.""" - # Invalid mode choice should raise SystemExit - test_args = ["profiling.sampling.sample", "--mode", "invalid", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("invalid choice", error_msg) - - def test_frames_filtered_with_skip_idle(self): - """Test that frames are actually filtered when skip_idle=True.""" - # Import thread status flags - try: - from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU - except ImportError: - THREAD_STATUS_HAS_GIL = (1 << 0) - THREAD_STATUS_ON_CPU = (1 << 1) - - # Create mock frames with different thread statuses - class MockThreadInfoWithStatus: - def __init__(self, thread_id, frame_info, status): - self.thread_id = thread_id - self.frame_info = frame_info - self.status = status - - # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread - ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU - IDLE_STATUS = 0 # Neither has GIL nor on CPU - - test_frames = [ - MockInterpreterInfo(0, [ - MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], ACTIVE_STATUS), - MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], IDLE_STATUS), - MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], ACTIVE_STATUS), - ]) - ] - - # Test with skip_idle=True - should only process running threads - collector_skip = PstatsCollector(sample_interval_usec=1000, skip_idle=True) - collector_skip.collect(test_frames) - - # Should only have functions from running threads (status 0) - active1_key = ("active1.py", 10, "active_func1") - active2_key = ("active2.py", 30, "active_func2") - idle_key = ("idle.py", 20, "idle_func") - - self.assertIn(active1_key, collector_skip.result) - self.assertIn(active2_key, collector_skip.result) - self.assertNotIn(idle_key, collector_skip.result) # Idle thread should be filtered out - - # Test with skip_idle=False - should process all threads - collector_no_skip = PstatsCollector(sample_interval_usec=1000, skip_idle=False) - collector_no_skip.collect(test_frames) - - # Should have functions from all threads - self.assertIn(active1_key, collector_no_skip.result) - self.assertIn(active2_key, collector_no_skip.result) - self.assertIn(idle_key, collector_no_skip.result) # Idle thread should be included - - @requires_subprocess() - def test_cpu_mode_integration_filtering(self): - """Integration test: CPU mode should only capture active threads, not idle ones.""" - # Script with one mostly-idle thread and one CPU-active thread - cpu_vs_idle_script = ''' -import time -import threading - -cpu_ready = threading.Event() - -def idle_worker(): - time.sleep(999999) - -def cpu_active_worker(): - cpu_ready.set() - x = 1 - while True: - x += 1 - -def main(): - # Start both threads - idle_thread = threading.Thread(target=idle_worker) - cpu_thread = threading.Thread(target=cpu_active_worker) - idle_thread.start() - cpu_thread.start() - - # Wait for CPU thread to be running, then signal test - cpu_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() - -''' - with test_subprocess(cpu_vs_idle_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=1, # CPU mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - cpu_mode_output = captured_output.getvalue() - - # Test wall-clock mode (mode=0) - should capture both functions - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=0, # Wall-clock mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - wall_mode_output = captured_output.getvalue() - - # Verify both modes captured samples - self.assertIn("Captured", cpu_mode_output) - self.assertIn("samples", cpu_mode_output) - self.assertIn("Captured", wall_mode_output) - self.assertIn("samples", wall_mode_output) - - # CPU mode should strongly favor cpu_active_worker over mostly_idle_worker - self.assertIn("cpu_active_worker", cpu_mode_output) - self.assertNotIn("idle_worker", cpu_mode_output) - - # Wall-clock mode should capture both types of work - self.assertIn("cpu_active_worker", wall_mode_output) - self.assertIn("idle_worker", wall_mode_output) - - def test_cpu_mode_with_no_samples(self): - """Test that CPU mode handles no samples gracefully when no samples are collected.""" - # Mock a collector that returns empty stats - mock_collector = mock.MagicMock() - mock_collector.stats = {} - mock_collector.create_stats = mock.MagicMock() - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - mock.patch("profiling.sampling.sample.PstatsCollector", return_value=mock_collector), - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler_class, - ): - mock_profiler = mock.MagicMock() - mock_profiler_class.return_value = mock_profiler - - profiling.sampling.sample.sample( - 12345, # dummy PID - duration_sec=0.5, - sample_interval_usec=5000, - mode=1, # CPU mode - show_summary=False, - all_threads=True, - ) - - output = captured_output.getvalue() - - # Should see the "No samples were collected" message - self.assertIn("No samples were collected", output) - self.assertIn("CPU mode", output) - - -class TestGilModeFiltering(unittest.TestCase): - """Test GIL mode filtering functionality (--mode=gil).""" - - def test_gil_mode_validation(self): - """Test that CLI accepts gil mode choice correctly.""" - test_args = ["profiling.sampling.sample", "--mode", "gil", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - try: - profiling.sampling.sample.main() - except SystemExit: - pass # Expected due to invalid PID - - # Should have attempted to call sample with mode=2 (GIL mode) - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["mode"], 2) # PROFILING_MODE_GIL - - def test_gil_mode_sample_function_call(self): - """Test that sample() function correctly uses GIL mode.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.PstatsCollector") as mock_collector, - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Mock the collector instance - mock_collector_instance = mock.Mock() - mock_collector.return_value = mock_collector_instance - - # Call sample with GIL mode and a filename to avoid pstats creation - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - duration_sec=1, - sample_interval_usec=1000, - filename="test_output.txt", - ) - - # Verify SampleProfiler was created with correct mode - mock_profiler.assert_called_once() - call_args = mock_profiler.call_args - self.assertEqual(call_args[1]['mode'], 2) # mode parameter - - # Verify profiler.sample was called - mock_instance.sample.assert_called_once() - - # Verify collector.export was called since we provided a filename - mock_collector_instance.export.assert_called_once_with("test_output.txt") - - def test_gil_mode_collector_configuration(self): - """Test that collectors are configured correctly for GIL mode.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.PstatsCollector") as mock_collector, - captured_stdout(), captured_stderr() - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Call sample with GIL mode - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - output_format="pstats", - ) - - # Verify collector was created with skip_idle=True (since mode != WALL) - mock_collector.assert_called_once() - call_args = mock_collector.call_args[1] - self.assertTrue(call_args['skip_idle']) - - def test_gil_mode_with_collapsed_format(self): - """Test GIL mode with collapsed stack format.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.CollapsedStackCollector") as mock_collector, - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Call sample with GIL mode and collapsed format - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - output_format="collapsed", - filename="test_output.txt", - ) - - # Verify collector was created with skip_idle=True - mock_collector.assert_called_once() - call_args = mock_collector.call_args[1] - self.assertTrue(call_args['skip_idle']) - - def test_gil_mode_cli_argument_parsing(self): - """Test CLI argument parsing for GIL mode with various options.""" - test_args = [ - "profiling.sampling.sample", - "--mode", "gil", - "--interval", "500", - "--duration", "5", - "-p", "12345" - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - try: - profiling.sampling.sample.main() - except SystemExit: - pass # Expected due to invalid PID - - # Verify all arguments were parsed correctly - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["mode"], 2) # GIL mode - self.assertEqual(call_args["sample_interval_usec"], 500) - self.assertEqual(call_args["duration_sec"], 5) - - @requires_subprocess() - def test_gil_mode_integration_behavior(self): - """Integration test: GIL mode should capture GIL-holding threads.""" - # Create a test script with GIL-releasing operations - gil_test_script = ''' -import time -import threading - -gil_ready = threading.Event() - -def gil_releasing_work(): - time.sleep(999999) - -def gil_holding_work(): - gil_ready.set() - x = 1 - while True: - x += 1 - -def main(): - # Start both threads - idle_thread = threading.Thread(target=gil_releasing_work) - cpu_thread = threading.Thread(target=gil_holding_work) - idle_thread.start() - cpu_thread.start() - - # Wait for GIL-holding thread to be running, then signal test - gil_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() -''' - with test_subprocess(gil_test_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=2, # GIL mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - gil_mode_output = captured_output.getvalue() - - # Test wall-clock mode for comparison - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=0.5, - sample_interval_usec=5000, - mode=0, # Wall-clock mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - wall_mode_output = captured_output.getvalue() - - # GIL mode should primarily capture GIL-holding work - # (Note: actual behavior depends on threading implementation) - self.assertIn("gil_holding_work", gil_mode_output) - - # Wall-clock mode should capture both types of work - self.assertIn("gil_holding_work", wall_mode_output) - - def test_mode_constants_are_defined(self): - """Test that all profiling mode constants are properly defined.""" - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_WALL, 0) - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_CPU, 1) - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_GIL, 2) - - def test_parse_mode_function(self): - """Test the _parse_mode function with all valid modes.""" - self.assertEqual(profiling.sampling.sample._parse_mode("wall"), 0) - self.assertEqual(profiling.sampling.sample._parse_mode("cpu"), 1) - self.assertEqual(profiling.sampling.sample._parse_mode("gil"), 2) - - # Test invalid mode raises KeyError - with self.assertRaises(KeyError): - profiling.sampling.sample._parse_mode("invalid") - - -@requires_subprocess() -@skip_if_not_supported -class TestGCFrameTracking(unittest.TestCase): - """Tests for GC frame tracking in the sampling profiler.""" - - @classmethod - def setUpClass(cls): - """Create a static test script with GC frames and CPU-intensive work.""" - cls.gc_test_script = ''' -import gc - -class ExpensiveGarbage: - """Class that triggers GC with expensive finalizer (callback).""" - def __init__(self): - self.cycle = self - - def __del__(self): - # CPU-intensive work in the finalizer callback - result = 0 - for i in range(100000): - result += i * i - if i % 1000 == 0: - result = result % 1000000 - -def main_loop(): - """Main loop that triggers GC with expensive callback.""" - while True: - ExpensiveGarbage() - gc.collect() - -if __name__ == "__main__": - main_loop() -''' - - def test_gc_frames_enabled(self): - """Test that GC frames appear when gc tracking is enabled.""" - with ( - test_subprocess(self.gc_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - native=False, - gc=True, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Should capture samples - self.assertIn("Captured", output) - self.assertIn("samples", output) - - # GC frames should be present - self.assertIn("", output) - - def test_gc_frames_disabled(self): - """Test that GC frames do not appear when gc tracking is disabled.""" - with ( - test_subprocess(self.gc_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - native=False, - gc=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Should capture samples - self.assertIn("Captured", output) - self.assertIn("samples", output) - - # GC frames should NOT be present - self.assertNotIn("", output) - - -@requires_subprocess() -@skip_if_not_supported -class TestNativeFrameTracking(unittest.TestCase): - """Tests for native frame tracking in the sampling profiler.""" - - @classmethod - def setUpClass(cls): - """Create a static test script with native frames and CPU-intensive work.""" - cls.native_test_script = ''' -import operator - -def main_loop(): - while True: - # Native code in the middle of the stack: - operator.call(inner) - -def inner(): - # Python code at the top of the stack: - for _ in range(1_000_0000): - pass - -if __name__ == "__main__": - main_loop() -''' - - def test_native_frames_enabled(self): - """Test that native frames appear when native tracking is enabled.""" - collapsed_file = tempfile.NamedTemporaryFile( - suffix=".txt", delete=False - ) - self.addCleanup(close_and_unlink, collapsed_file) - - with ( - test_subprocess(self.native_test_script) as subproc, - ): - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=collapsed_file.name, - output_format="collapsed", - sample_interval_usec=1000, - native=True, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(collapsed_file.name)) - self.assertGreater(os.path.getsize(collapsed_file.name), 0) - - # Check file format - with open(collapsed_file.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertGreater(len(lines), 0) - - stacks = [line.rsplit(" ", 1)[0] for line in lines] - - # Most samples should have native code in the middle of the stack: - self.assertTrue(any(";;" in stack for stack in stacks)) - - # No samples should have native code at the top of the stack: - self.assertFalse(any(stack.endswith(";") for stack in stacks)) - - def test_native_frames_disabled(self): - """Test that native frames do not appear when native tracking is disabled.""" - with ( - test_subprocess(self.native_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - output = captured_output.getvalue() - # Native frames should NOT be present: - self.assertNotIn("", output) - - -@requires_subprocess() -@skip_if_not_supported -class TestProcessPoolExecutorSupport(unittest.TestCase): - """ - Test that ProcessPoolExecutor works correctly with profiling.sampling. - """ - - def test_process_pool_executor_pickle(self): - # gh-140729: test use ProcessPoolExecutor.map() can sampling - test_script = ''' -import concurrent.futures - -def worker(x): - return x * 2 - -if __name__ == "__main__": - with concurrent.futures.ProcessPoolExecutor() as executor: - results = list(executor.map(worker, [1, 2, 3])) - print(f"Results: {results}") -''' - with os_helper.temp_dir() as temp_dir: - script = script_helper.make_script( - temp_dir, 'test_process_pool_executor_pickle', test_script - ) - with SuppressCrashReport(): - with script_helper.spawn_python( - "-m", "profiling.sampling.sample", - "-d", "5", - "-i", "100000", - script, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) as proc: - try: - stdout, stderr = proc.communicate(timeout=SHORT_TIMEOUT) - except subprocess.TimeoutExpired: - proc.kill() - stdout, stderr = proc.communicate() - - if "PermissionError" in stderr: - self.skipTest("Insufficient permissions for remote profiling") - - self.assertIn("Results: [2, 4, 6]", stdout) - self.assertNotIn("Can't pickle", stderr) -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/__init__.py b/Lib/test/test_profiling/test_sampling_profiler/__init__.py new file mode 100644 index 00000000000..616ae5b49f0 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/__init__.py @@ -0,0 +1,9 @@ +"""Tests for the sampling profiler (profiling.sampling).""" + +import os +from test.support import load_package_tests + + +def load_tests(*args): + """Load all tests from this subpackage.""" + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py new file mode 100644 index 00000000000..abd5a7377ad --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -0,0 +1,101 @@ +"""Helper utilities for sampling profiler tests.""" + +import contextlib +import socket +import subprocess +import sys +import unittest +from collections import namedtuple + +from test.support import SHORT_TIMEOUT +from test.support.socket_helper import find_unused_port +from test.support.os_helper import unlink + + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED # noqa: F401 + import _remote_debugging # noqa: F401 +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) +else: + import profiling.sampling # noqa: F401 + from profiling.sampling.sample import SampleProfiler # noqa: F401 + + +skip_if_not_supported = unittest.skipIf( + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), + "Test only runs on Linux, Windows and MacOS", +) + +SubprocessInfo = namedtuple("SubprocessInfo", ["process", "socket"]) + + +@contextlib.contextmanager +def test_subprocess(script): + """Context manager to create a test subprocess with socket synchronization. + + Args: + script: Python code to execute in the subprocess + + Yields: + SubprocessInfo: Named tuple with process and socket objects + """ + # Find an unused port for socket communication + port = find_unused_port() + + # Inject socket connection code at the beginning of the script + socket_code = f""" +import socket +_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +_test_sock.connect(('localhost', {port})) +_test_sock.sendall(b"ready") +""" + + # Combine socket code with user script + full_script = socket_code + script + + # Create server socket to wait for process to be ready + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + proc = subprocess.Popen( + [sys.executable, "-c", full_script], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + client_socket = None + try: + # Wait for process to connect and send ready signal + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + if response != b"ready": + raise RuntimeError( + f"Unexpected response from subprocess: {response}" + ) + + yield SubprocessInfo(proc, client_socket) + finally: + if client_socket is not None: + client_socket.close() + if proc.poll() is None: + proc.kill() + proc.wait() + + +def close_and_unlink(file): + """Close a file and unlink it from the filesystem.""" + file.close() + unlink(file.name) diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py new file mode 100644 index 00000000000..9f1cd5b83e0 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -0,0 +1,38 @@ +"""Mock classes for sampling profiler tests.""" + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__( + self, thread_id, frame_info, status=0 + ): # Default to THREAD_STATE_RUNNING (0) + self.thread_id = thread_id + self.frame_info = frame_info + self.status = status + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py new file mode 100644 index 00000000000..578fb51bc0c --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py @@ -0,0 +1,264 @@ +"""Tests for advanced sampling profiler features (GC tracking, native frames, ProcessPoolExecutor support).""" + +import io +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import ( + SHORT_TIMEOUT, + SuppressCrashReport, + os_helper, + requires_subprocess, + script_helper, +) + +from .helpers import close_and_unlink, skip_if_not_supported, test_subprocess + + +@requires_subprocess() +@skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + """Class that triggers GC with expensive finalizer (callback).""" + def __init__(self): + self.cycle = self + + def __del__(self): + # CPU-intensive work in the finalizer callback + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +def main_loop(): + """Main loop that triggers GC with expensive callback.""" + while True: + ExpensiveGarbage() + gc.collect() + +if __name__ == "__main__": + main_loop() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = """ +import operator + +def main_loop(): + while True: + # Native code in the middle of the stack: + operator.call(inner) + +def inner(): + # Python code at the top of the stack: + for _ in range(1_000_0000): + pass + +if __name__ == "__main__": + main_loop() +""" + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.native_test_script) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=1000, + native=True, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = """ +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +""" + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, "test_process_pool_executor_pickle", test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", + "profiling.sampling.sample", + "-d", + "5", + "-i", + "100000", + script, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + stdout, stderr = proc.communicate( + timeout=SHORT_TIMEOUT + ) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() + + if "PermissionError" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py new file mode 100644 index 00000000000..5833920d1b9 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -0,0 +1,664 @@ +"""Tests for sampling profiler CLI argument parsing and functionality.""" + +import io +import subprocess +import sys +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import is_emscripten + + +class TestSampleProfilerCLI(unittest.TestCase): + def _setup_sync_mocks(self, mock_socket, mock_popen): + """Helper to set up socket and process mocks for coordinator tests.""" + # Mock the sync socket with context manager support + mock_sock_instance = mock.MagicMock() + mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) + + # Mock the connection with context manager support + mock_conn = mock.MagicMock() + mock_conn.recv.return_value = b"ready" + mock_conn.__enter__.return_value = mock_conn + mock_conn.__exit__.return_value = None + + # Mock accept() to return (connection, address) and support indexing + mock_accept_result = mock.MagicMock() + mock_accept_result.__getitem__.return_value = ( + mock_conn # [0] returns the connection + ) + mock_sock_instance.accept.return_value = mock_accept_result + + # Mock socket with context manager support + mock_sock_instance.__enter__.return_value = mock_sock_instance + mock_sock_instance.__exit__.return_value = None + mock_socket.return_value = mock_sock_instance + + # Mock the subprocess + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.poll.return_value = None + mock_popen.return_value = mock_process + return mock_process + + def _verify_coordinator_command(self, mock_popen, expected_target_args): + """Helper to verify the coordinator command was called correctly.""" + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual(coordinator_cmd[5:], expected_target_args) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_argument_parsing(self): + test_args = ["profiling.sampling.sample", "-m", "mymodule"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort (sort_value from args.sort) + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_arguments(self): + test_args = [ + "profiling.sampling.sample", + "-m", + "mymodule", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag") + ) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_argument_parsing(self): + test_args = ["profiling.sampling.sample", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("myscript.py",)) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_arguments(self): + test_args = [ + "profiling.sampling.sample", + "myscript.py", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + # Use the helper to set up mocks consistently + mock_process = self._setup_sync_mocks(mock_socket, mock_popen) + # Override specific behavior for this test + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + + profiling.sampling.sample.main() + + # Verify the coordinator command was called + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual( + coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag") + ) + + def test_cli_mutually_exclusive_pid_module(self): + test_args = [ + "profiling.sampling.sample", + "-p", + "12345", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("not allowed with argument", error_msg) + + def test_cli_mutually_exclusive_pid_script(self): + test_args = ["profiling.sampling.sample", "-p", "12345", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("only one target type can be specified", error_msg) + + def test_cli_no_target_specified(self): + test_args = ["profiling.sampling.sample", "-d", "5"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("one of the arguments", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_profiler_options(self): + test_args = [ + "profiling.sampling.sample", + "-i", + "1000", + "-d", + "30", + "-a", + "--sort-tottime", + "-l", + "20", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=1, # sort-tottime + sample_interval_usec=1000, + duration_sec=30, + filename=None, + all_threads=True, + limit=20, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_profiler_options(self): + """Test script with various profiler options.""" + test_args = [ + "profiling.sampling.sample", + "-i", + "2000", + "-d", + "60", + "--collapsed", + "-o", + "output.txt", + "myscript.py", + "scriptarg", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("myscript.py", "scriptarg") + ) + # Verify profiler options were passed correctly + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort + sample_interval_usec=2000, + duration_sec=60, + filename="output.txt", + all_threads=False, + limit=15, + show_summary=True, + output_format="collapsed", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + def test_cli_empty_module_name(self): + test_args = ["profiling.sampling.sample", "-m"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("argument -m/--module: expected one argument", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_long_module_option(self): + test_args = [ + "profiling.sampling.sample", + "--module", + "mymodule", + "arg1", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1") + ) + + def test_cli_complex_script_arguments(self): + test_args = [ + "profiling.sampling.sample", + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch( + "profiling.sampling.sample._run_with_sync" + ) as mock_run_with_sync, + ): + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + mock_process.poll.return_value = None + mock_run_with_sync.return_value = mock_process + + profiling.sampling.sample.main() + + mock_run_with_sync.assert_called_once_with( + ( + sys.executable, + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ) + ) + + def test_cli_collapsed_format_validation(self): + """Test that CLI properly validates incompatible options with collapsed format.""" + test_cases = [ + # Test sort options are invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-nsamples", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-tottime", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-cumtime", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-sample-pct", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-cumul-pct", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-name", + "-p", + "12345", + ], + "sort", + ), + # Test limit option is invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "-l", + "20", + "-p", + "12345", + ], + "limit", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--limit", + "20", + "-p", + "12345", + ], + "limit", + ), + # Test no-summary option is invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--no-summary", + "-p", + "12345", + ], + "summary", + ), + ] + + for test_args, expected_error_keyword in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error code + error_msg = mock_stderr.getvalue() + self.assertIn("error:", error_msg) + self.assertIn("--pstats format", error_msg) + + def test_cli_default_collapsed_filename(self): + """Test that collapsed format gets a default filename when not specified.""" + test_args = ["profiling.sampling.sample", "--collapsed", "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + # Check that filename was set to default collapsed format + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["output_format"], "collapsed") + self.assertEqual(call_args["filename"], "collapsed.12345.txt") + + def test_cli_custom_output_filenames(self): + """Test custom output filenames for both formats.""" + test_cases = [ + ( + [ + "profiling.sampling.sample", + "--pstats", + "-o", + "custom.pstats", + "-p", + "12345", + ], + "custom.pstats", + "pstats", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "-o", + "custom.txt", + "-p", + "12345", + ], + "custom.txt", + "collapsed", + ), + ] + + for test_args, expected_filename, expected_format in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["filename"], expected_filename) + self.assertEqual(call_args["output_format"], expected_format) + + def test_cli_missing_required_arguments(self): + """Test that CLI requires PID argument.""" + with ( + mock.patch("sys.argv", ["profiling.sampling.sample"]), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profiling.sampling.sample.main() + + def test_cli_mutually_exclusive_format_options(self): + """Test that pstats and collapsed options are mutually exclusive.""" + with ( + mock.patch( + "sys.argv", + [ + "profiling.sampling.sample", + "--pstats", + "--collapsed", + "-p", + "12345", + ], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profiling.sampling.sample.main() + + def test_argument_parsing_basic(self): + test_args = ["profiling.sampling.sample", "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once_with( + 12345, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + sort=2, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + def test_sort_options(self): + sort_options = [ + ("--sort-nsamples", 0), + ("--sort-tottime", 1), + ("--sort-cumtime", 2), + ("--sort-sample-pct", 3), + ("--sort-cumul-pct", 4), + ("--sort-name", -1), + ] + + for option, expected_sort_value in sort_options: + test_args = ["profiling.sampling.sample", option, "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual( + call_args["sort"], + expected_sort_value, + ) + mock_sample.reset_mock() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py new file mode 100644 index 00000000000..4a24256203c --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -0,0 +1,896 @@ +"""Tests for sampling profiler collector components.""" + +import json +import marshal +import os +import tempfile +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import ( + CollapsedStackCollector, + FlamegraphCollector, + ) + from profiling.sampling.gecko_collector import GeckoCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import captured_stdout, captured_stderr + +from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo +from .helpers import close_and_unlink + + +class TestSampleProfilerComponents(unittest.TestCase): + """Unit tests for individual profiler components.""" + + def test_mock_frame_info_with_empty_and_unicode_values(self): + """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" + # Test with empty strings + frame = MockFrameInfo("", 0, "") + self.assertEqual(frame.filename, "") + self.assertEqual(frame.lineno, 0) + self.assertEqual(frame.funcname, "") + self.assertIn("filename=''", repr(frame)) + + # Test with unicode characters + frame = MockFrameInfo("文件.py", 42, "函数名") + self.assertEqual(frame.filename, "文件.py") + self.assertEqual(frame.funcname, "函数名") + + # Test with very long names + long_filename = "x" * 1000 + ".py" + long_funcname = "func_" + "x" * 1000 + frame = MockFrameInfo(long_filename, 999999, long_funcname) + self.assertEqual(frame.filename, long_filename) + self.assertEqual(frame.lineno, 999999) + self.assertEqual(frame.funcname, long_funcname) + + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): + """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" + # Test with zero interval + collector = PstatsCollector(sample_interval_usec=0) + self.assertEqual(collector.sample_interval_usec, 0) + + # Test with very large interval + collector = PstatsCollector(sample_interval_usec=1000000000) + self.assertEqual(collector.sample_interval_usec, 1000000000) + + # Test collecting empty frames list + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect([]) + self.assertEqual(len(collector.result), 0) + + # Test collecting frames with None thread id + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])], + ) + ] + collector.collect(test_frames) + # Should still process the frames + self.assertEqual(len(collector.result), 1) + + # Test collecting duplicate frames in same sample + test_frames = [ + MockInterpreterInfo( + 0, # interpreter_id + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 10, "func1"), # Duplicate + ], + ) + ], + ) + ] + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(test_frames) + # Should count both occurrences + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 + ) + + def test_pstats_collector_single_frame_stacks(self): + """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test with exactly one frame (should trigger the <= 1 condition) + single_frame = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("single.py", 10, "single_func")] + ) + ], + ) + ] + collector.collect(single_frame) + + # Should record the single frame with inline call + self.assertEqual(len(collector.result), 1) + single_key = ("single.py", 10, "single_func") + self.assertIn(single_key, collector.result) + self.assertEqual(collector.result[single_key]["direct_calls"], 1) + self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) + + # Test with empty frames (should also trigger <= 1 condition) + empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] + collector.collect(empty_frames) + + # Should not add any new entries + self.assertEqual( + len(collector.result), 1 + ) # Still just the single frame + + # Test mixed single and multi-frame stacks + mixed_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("single2.py", 20, "single_func2")], + ), # Single frame + MockThreadInfo( + 2, + [ # Multi-frame stack + MockFrameInfo("multi.py", 30, "multi_func1"), + MockFrameInfo("multi.py", 40, "multi_func2"), + ], + ), + ], + ), + ] + collector.collect(mixed_frames) + + # Should have recorded all functions + self.assertEqual( + len(collector.result), 4 + ) # single + single2 + multi1 + multi2 + + # Verify single frame handling + single2_key = ("single2.py", 20, "single_func2") + self.assertIn(single2_key, collector.result) + self.assertEqual(collector.result[single2_key]["direct_calls"], 1) + self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) + + # Verify multi-frame handling still works + multi1_key = ("multi.py", 30, "multi_func1") + multi2_key = ("multi.py", 40, "multi_func2") + self.assertIn(multi1_key, collector.result) + self.assertIn(multi2_key, collector.result) + self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) + self.assertEqual( + collector.result[multi2_key]["cumulative_calls"], 1 + ) # Called from multi1 + + def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): + """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" + collector = CollapsedStackCollector() + + # Test with empty frames + collector.collect([]) + self.assertEqual(len(collector.stack_counter), 0) + + # Test with single frame stack + test_frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("file.py", 10, "func")])] + ) + ] + collector.collect(test_frames) + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + self.assertEqual(path, (("file.py", 10, "func"),)) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + # Test with very deep stack + deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] + test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] + collector = CollapsedStackCollector() + collector.collect(test_frames) + # One aggregated path with 100 frames (reversed) + (((path_tuple, thread_id),),) = (collector.stack_counter.keys(),) + self.assertEqual(len(path_tuple), 100) + self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) + self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) + self.assertEqual(thread_id, 1) + + def test_pstats_collector_basic(self): + """Test basic PstatsCollector functionality.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test empty state + self.assertEqual(len(collector.result), 0) + self.assertEqual(len(collector.stats), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + + # Should have recorded calls for both functions + self.assertEqual(len(collector.result), 2) + self.assertIn(("file.py", 10, "func1"), collector.result) + self.assertIn(("file.py", 20, "func2"), collector.result) + + # Top-level function should have direct call + self.assertEqual( + collector.result[("file.py", 10, "func1")]["direct_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 + ) + + # Calling function should have cumulative call but no direct calls + self.assertEqual( + collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 20, "func2")]["direct_calls"], 0 + ) + + def test_pstats_collector_create_stats(self): + """Test PstatsCollector stats creation.""" + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + collector.collect(test_frames) # Collect twice + + collector.create_stats() + + # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) + func1_stats = collector.stats[("file.py", 10, "func1")] + self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) + self.assertEqual(func1_stats[1], 2) # cumulative_calls + self.assertEqual( + func1_stats[2], 2.0 + ) # tt (total time - 2 samples * 1 sec) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + func2_stats = collector.stats[("file.py", 20, "func2")] + self.assertEqual( + func2_stats[0], 0 + ) # direct_calls (never top of stack) + self.assertEqual( + func2_stats[1], 2 + ) # cumulative_calls (appears in stack) + self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) + self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) + + def test_collapsed_stack_collector_basic(self): + collector = CollapsedStackCollector() + + # Test empty state + self.assertEqual(len(collector.stack_counter), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Should store one reversed path + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) + self.assertEqual(path, expected_tree) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + def test_collapsed_stack_collector_export(self): + collapsed_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, collapsed_out) + + collector = CollapsedStackCollector() + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + with captured_stdout(), captured_stderr(): + collector.export(collapsed_out.name) + # Check file contents + with open(collapsed_out.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertEqual(len(lines), 2) # Two unique stacks + + # Check collapsed format: tid:X;file:func:line;file:func:line count + stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" + stack2_expected = "tid:1;other.py:other_func:5 1" + + self.assertIn(stack1_expected, lines) + self.assertIn(stack2_expected, lines) + + def test_flamegraph_collector_basic(self): + """Test basic FlamegraphCollector functionality.""" + collector = FlamegraphCollector() + + # Empty collector should produce 'No Data' + data = collector._convert_to_flamegraph_format() + # With string table, name is now an index - resolve it using the strings array + strings = data.get("strings", []) + name_index = data.get("name", 0) + resolved_name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIn(resolved_name, ("No Data", "No significant data")) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Convert and verify structure: func2 -> func1 with counts = 1 + data = collector._convert_to_flamegraph_format() + # Expect promotion: root is the single child (func2), with func1 as its only child + strings = data.get("strings", []) + name_index = data.get("name", 0) + name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIsInstance(name, str) + self.assertTrue(name.startswith("Program Root: ")) + self.assertIn("func2 (file.py:20)", name) # formatted name + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + child_name_index = child.get("name", 0) + child_name = ( + strings[child_name_index] + if isinstance(child_name_index, int) + and 0 <= child_name_index < len(strings) + else str(child_name_index) + ) + self.assertIn("func1 (file.py:10)", child_name) # formatted name + self.assertEqual(child["value"], 1) + + def test_flamegraph_collector_export(self): + """Test flamegraph HTML export functionality.""" + flamegraph_out = tempfile.NamedTemporaryFile( + suffix=".html", delete=False + ) + self.addCleanup(close_and_unlink, flamegraph_out) + + collector = FlamegraphCollector() + + # Create some test data (use Interpreter/Thread objects like runtime) + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export flamegraph + with captured_stdout(), captured_stderr(): + collector.export(flamegraph_out.name) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(flamegraph_out.name)) + self.assertGreater(os.path.getsize(flamegraph_out.name), 0) + + # Check file contains HTML content + with open(flamegraph_out.name, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("", content.lower()) + self.assertIn(" 0) + self.assertGreater(mock_collector.collect.call_count, 0) + self.assertLessEqual(mock_collector.collect.call_count, 3) + + def test_sample_profiler_missed_samples_warning(self): + """Test that the profiler warns about missed samples when sampling is too slow.""" + + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + # Use very short interval that we'll miss + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) # 1ms interval + + mock_collector = mock.MagicMock() + + # Simulate slow sampling where we miss many samples + times = [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + ] # Extra time points to avoid StopIteration + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.5) + + result = output.getvalue() + + # Should warn about missed samples + self.assertIn("Warning: missed", result) + self.assertIn("samples from the expected total", result) + + +@force_not_colorized_test_class +class TestPrintSampledStats(unittest.TestCase): + """Test the print_sampled_stats function.""" + + def setUp(self): + """Set up test data.""" + # Mock stats data + self.mock_stats = mock.MagicMock() + self.mock_stats.stats = { + ("file1.py", 10, "func1"): ( + 100, + 100, + 0.5, + 0.5, + {}, + ), # cc, nc, tt, ct, callers + ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), + ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), + ("file4.py", 40, "func4"): ( + 10, + 10, + 0.001, + 0.001, + {}, + ), # millisecond range + ("file5.py", 50, "func5"): ( + 5, + 5, + 0.000001, + 0.000002, + {}, + ), # microsecond range + } + + def test_print_sampled_stats_basic(self): + """Test basic print_sampled_stats functionality.""" + + # Capture output + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Check header is present + self.assertIn("Profile Stats:", result) + self.assertIn("nsamples", result) + self.assertIn("tottime", result) + self.assertIn("cumtime", result) + + # Check functions are present + self.assertIn("func1", result) + self.assertIn("func2", result) + self.assertIn("func3", result) + + def test_print_sampled_stats_sorting(self): + """Test different sorting options.""" + + # Test sort by calls + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=0, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header) + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (200 calls) + self.assertIn("func3", data_lines[0]) + + # Test sort by time + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=1, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (1.5s time) + self.assertIn("func3", data_lines[0]) + + def test_print_sampled_stats_limit(self): + """Test limiting output rows.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, limit=2, sample_interval_usec=100 + ) + + result = output.getvalue() + + # Count function entries in the main stats section (not in summary) + lines = result.split("\n") + # Find where the main stats section ends (before summary) + main_section_lines = [] + for line in lines: + if "Summary of Interesting Functions:" in line: + break + main_section_lines.append(line) + + # Count function entries only in main section + func_count = sum( + 1 + for line in main_section_lines + if "func" in line and ".py" in line + ) + self.assertEqual(func_count, 2) + + def test_print_sampled_stats_time_units(self): + """Test proper time unit selection.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use seconds for the header since max time is > 1s + self.assertIn("tottime (s)", result) + self.assertIn("cumtime (s)", result) + + # Test with only microsecond-range times + micro_stats = mock.MagicMock() + micro_stats.stats = { + ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(micro_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use microseconds + self.assertIn("tottime (μs)", result) + self.assertIn("cumtime (μs)", result) + + def test_print_sampled_stats_summary(self): + """Test summary section generation.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Check summary sections are present + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn( + "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", + result, + ) + self.assertIn( + "Functions with Highest Call Frequency (Indirect Calls):", result + ) + self.assertIn( + "Functions with Highest Call Magnification (Cumulative/Direct):", + result, + ) + + def test_print_sampled_stats_no_summary(self): + """Test disabling summary output.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=False, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Summary should not be present + self.assertNotIn("Summary of Interesting Functions:", result) + + def test_print_sampled_stats_empty_stats(self): + """Test with empty stats.""" + + empty_stats = mock.MagicMock() + empty_stats.stats = {} + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(empty_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should still print header + self.assertIn("Profile Stats:", result) + + def test_print_sampled_stats_sample_percentage_sorting(self): + """Test sample percentage sorting options.""" + + # Add a function with high sample percentage (more direct calls than func3's 200) + self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( + 300, # direct calls (higher than func3's 200) + 300, # cumulative calls + 1.0, # total time + 1.0, # cumulative time + {}, + ) + + # Test sort by sample percentage + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=3, sample_interval_usec=100 + ) # sample percentage + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if ".py" in l and "func" in l] + # expensive_func should be first (highest sample percentage) + self.assertIn("expensive_func", data_lines[0]) + + def test_print_sampled_stats_with_recursive_calls(self): + """Test print_sampled_stats with recursive calls where nc != cc.""" + + # Create stats with recursive calls (nc != cc) + recursive_stats = mock.MagicMock() + recursive_stats.stats = { + # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function + ("recursive.py", 10, "factorial"): ( + 5, # direct_calls + 10, # cumulative_calls (appears more times in stack due to recursion) + 0.5, + 0.6, + {}, + ), + ("normal.py", 20, "normal_func"): ( + 3, # direct_calls + 3, # cumulative_calls (same as direct for non-recursive) + 0.2, + 0.2, + {}, + ), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(recursive_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should display recursive calls as "5/10" format + self.assertIn("5/10", result) # nc/cc format for recursive calls + self.assertIn("3", result) # just nc for non-recursive calls + self.assertIn("factorial", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_zero_call_counts(self): + """Test print_sampled_stats with zero call counts to trigger division protection.""" + + # Create stats with zero call counts + zero_stats = mock.MagicMock() + zero_stats.stats = { + ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls + ("file.py", 20, "normal_func"): ( + 5, + 5, + 0.1, + 0.1, + {}, + ), # Normal function + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(zero_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should handle zero call counts gracefully + self.assertIn("zero_calls", result) + self.assertIn("zero_calls", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_sort_by_name(self): + """Test sort by function name option.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=-1, sample_interval_usec=100 + ) # sort by name + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header and summary) + # Data lines start with whitespace and numbers, and contain filename:lineno(function) + data_lines = [] + for line in lines: + # Skip header lines and summary sections + if ( + line.startswith(" ") + and "(" in line + and ")" in line + and not line.startswith( + " 1." + ) # Skip summary lines that start with times + and not line.startswith( + " 0." + ) # Skip summary lines that start with times + and not "per call" in line # Skip summary lines + and not "calls" in line # Skip summary lines + and not "total time" in line # Skip summary lines + and not "cumulative time" in line + ): # Skip summary lines + data_lines.append(line) + + # Extract just the function names for comparison + func_names = [] + import re + + for line in data_lines: + # Function name is between the last ( and ), accounting for ANSI color codes + match = re.search(r"\(([^)]+)\)$", line) + if match: + func_name = match.group(1) + # Remove ANSI color codes + func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) + func_names.append(func_name) + + # Verify we extracted function names and they are sorted + self.assertGreater( + len(func_names), 0, "Should have extracted some function names" + ) + self.assertEqual( + func_names, + sorted(func_names), + f"Function names {func_names} should be sorted alphabetically", + ) + + def test_print_sampled_stats_with_zero_time_functions(self): + """Test summary sections with functions that have zero time.""" + + # Create stats with zero-time functions + zero_time_stats = mock.MagicMock() + zero_time_stats.stats = { + ("file1.py", 10, "zero_time_func"): ( + 5, + 5, + 0.0, + 0.0, + {}, + ), # Zero time + ("file2.py", 20, "normal_func"): ( + 3, + 3, + 0.1, + 0.1, + {}, + ), # Normal time + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + zero_time_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle zero-time functions gracefully in summary + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn("zero_time_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_malformed_qualified_names(self): + """Test summary generation with function names that don't contain colons.""" + + # Create stats with function names that would create malformed qualified names + malformed_stats = mock.MagicMock() + malformed_stats.stats = { + # Function name without clear module separation + ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), + ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), + ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + malformed_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle malformed names gracefully in summary aggregation + self.assertIn("Summary of Interesting Functions:", result) + # All function names should appear somewhere in the output + self.assertIn("func", result) + self.assertIn("empty_filename_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_recursive_call_stats_creation(self): + """Test create_stats with recursive call data to trigger total_rec_calls branch.""" + collector = PstatsCollector(sample_interval_usec=1000000) # 1 second + + # Simulate recursive function data where total_rec_calls would be set + # We need to manually manipulate the collector result to test this branch + collector.result = { + ("recursive.py", 10, "factorial"): { + "total_rec_calls": 3, # Non-zero recursive calls + "direct_calls": 5, + "cumulative_calls": 10, + }, + ("normal.py", 20, "normal_func"): { + "total_rec_calls": 0, # Zero recursive calls + "direct_calls": 2, + "cumulative_calls": 5, + }, + } + + collector.create_stats() + + # Check that recursive calls are handled differently from non-recursive + factorial_stats = collector.stats[("recursive.py", 10, "factorial")] + normal_stats = collector.stats[("normal.py", 20, "normal_func")] + + # factorial should use cumulative_calls (10) as nc + self.assertEqual( + factorial_stats[1], 10 + ) # nc should be cumulative_calls + self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls + + # normal_func should use cumulative_calls as nc + self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls + self.assertEqual(normal_stats[0], 2) # cc should be direct_calls diff --git a/Makefile.pre.in b/Makefile.pre.in index dd28ff5d2a3..59c3c808794 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2692,6 +2692,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_pathlib/support \ test/test_peg_generator \ test/test_profiling \ + test/test_profiling/test_sampling_profiler \ test/test_pydoc \ test/test_pyrepl \ test/test_string \ From 4695ec109d07c9bfd9eb7d91d6285c974a4331a7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Nov 2025 16:33:52 +0100 Subject: [PATCH 234/417] gh-138189: Link references to type slots (GH-141410) Link references to type slots --- Doc/c-api/structures.rst | 8 +++----- Doc/c-api/type.rst | 16 ++++++++-------- Doc/c-api/typeobj.rst | 2 +- Doc/howto/isolating-extensions.rst | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 414dfdc84e6..b4e7cb1d77e 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -698,14 +698,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index b608f815160..c7946e3190f 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -383,8 +383,8 @@ The following functions and structs are used to create The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -590,9 +590,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -611,7 +611,7 @@ The following functions and structs are used to create If it is not possible to switch to a ``MANAGED`` flag (for example, for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. + offset in :c:data:`Py_tp_members`. See :ref:`PyMemberDef documentation ` for details. @@ -639,7 +639,7 @@ The following functions and structs are used to create .. versionchanged:: 3.14 The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation + using :c:data:`Py_tp_vectorcall`. See the field's documentation for details. .. c:member:: void *pfunc @@ -649,7 +649,7 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 34d19acdf17..49fe02d919d 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2273,7 +2273,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 7da6dc8a397..6092c75f48f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -353,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of From 600f3feb234219c9a9998e30ea653a2afb1f8116 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 18 Nov 2025 17:13:13 +0100 Subject: [PATCH 235/417] gh-141070: Add PyUnstable_Object_Dump() function (#141072) * Promote _PyObject_Dump() as a public function. * Keep _PyObject_Dump() alias to PyUnstable_Object_Dump() for backward compatibility. * Replace _PyObject_Dump() with PyUnstable_Object_Dump(). Co-authored-by: Peter Bierma Co-authored-by: Kumar Aditya Co-authored-by: Petr Viktorin --- Doc/c-api/object.rst | 29 +++++++++++ Doc/whatsnew/3.15.rst | 12 +++-- Include/cpython/object.h | 14 +++-- .../pycore_global_objects_fini_generated.h | 2 +- Lib/test/test_capi/test_object.py | 52 +++++++++++++++++++ ...-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst | 2 + Modules/_testcapi/object.c | 25 +++++++++ Objects/object.c | 4 +- Objects/unicodeobject.c | 3 +- Python/gc.c | 2 +- Python/pythonrun.c | 8 +-- 11 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 96353266ac7..76971c46c16 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyUnstable_Object_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: next + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 24cc7e2d7eb..5a98297d3f8 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1084,19 +1084,23 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python thread state. (Contributed by Victor Stinner in :gh:`139653`.) -* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating - a module from a *spec* and *initfunc*. - (Contributed by Itamar Oren in :gh:`116146`.) - Changed C APIs -------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d64298232e7..130a105de42 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -295,7 +295,10 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_Dump(PyObject *); + +// Alias for backward compatibility +#define _PyObject_Dump PyUnstable_Object_Dump PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); @@ -387,10 +390,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyUnstable_Object_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ecef4364cc3..c3968aff8f3 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); } } #endif diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index d4056727d07..c5040913e9e 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,4 +1,5 @@ import enum +import os import sys import textwrap import unittest @@ -13,6 +14,9 @@ _testcapi = import_helper.import_module('_testcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') +NULL = None +STDERR_FD = 2 + class Constant(enum.IntEnum): Py_CONSTANT_NONE = 0 @@ -247,5 +251,53 @@ def func(x): func(object()) + def pyobject_dump(self, obj, release_gil=False): + pyobject_dump = _testcapi.pyobject_dump + + try: + old_stderr = os.dup(STDERR_FD) + except OSError as exc: + # os.dup(STDERR_FD) is not supported on WASI + self.skipTest(f"os.dup() failed with {exc!r}") + + filename = os_helper.TESTFN + try: + try: + with open(filename, "wb") as fp: + fd = fp.fileno() + os.dup2(fd, STDERR_FD) + pyobject_dump(obj, release_gil) + finally: + os.dup2(old_stderr, STDERR_FD) + os.close(old_stderr) + + with open(filename) as fp: + return fp.read().rstrip() + finally: + os_helper.unlink(filename) + + def test_pyobject_dump(self): + # test string object + str_obj = 'test string' + output = self.pyobject_dump(str_obj) + hex_regex = r'(0x)?[0-9a-fA-F]+' + regex = ( + fr"object address : {hex_regex}\n" + r"object refcount : [0-9]+\n" + fr"object type : {hex_regex}\n" + r"object type name: str\n" + r"object repr : 'test string'" + ) + self.assertRegex(output, regex) + + # release the GIL + output = self.pyobject_dump(str_obj, release_gil=True) + self.assertRegex(output, regex) + + # test NULL object + output = self.pyobject_dump(NULL) + self.assertRegex(output, r'') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst new file mode 100644 index 00000000000..39cfcf73404 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. It should +only be used for debugging. Patch by Victor Stinner. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 798ef97c495..4c9632c07a9 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -485,6 +485,30 @@ is_uniquely_referenced(PyObject *self, PyObject *op) } +static PyObject * +pyobject_dump(PyObject *self, PyObject *args) +{ + PyObject *op; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "O|i", &op, &release_gil)) { + return NULL; + } + NULLABLE(op); + + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + PyUnstable_Object_Dump(op); + Py_END_ALLOW_THREADS + + } + else { + PyUnstable_Object_Dump(op); + } + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, @@ -511,6 +535,7 @@ static PyMethodDef test_methods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, + {"pyobject_dump", pyobject_dump, METH_VARARGS}, {NULL}, }; diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2..0a80c6edcf1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -713,7 +713,7 @@ _PyObject_IsFreed(PyObject *op) /* For debugging convenience. See Misc/gdbinit for some useful gdb hooks */ void -_PyObject_Dump(PyObject* op) +PyUnstable_Object_Dump(PyObject* op) { if (_PyObject_IsFreed(op)) { /* It seems like the object memory has been freed: @@ -3150,7 +3150,7 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, /* This might succeed or fail, but we're about to abort, so at least try to provide any extra info we can: */ - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); fprintf(stderr, "\n"); fflush(stderr); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4e8c132327b..7f9f75126a9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -547,7 +547,8 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) } /* Disable checks during Python finalization. For example, it allows to - call _PyObject_Dump() during finalization for debugging purpose. */ + * call PyUnstable_Object_Dump() during finalization for debugging purpose. + */ if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } diff --git a/Python/gc.c b/Python/gc.c index 064f9406e0a..27364ecfdcd 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2237,7 +2237,7 @@ _PyGC_Fini(PyInterpreterState *interp) void _PyGC_Dump(PyGC_Head *g) { - _PyObject_Dump(FROM_GC(g)); + PyUnstable_Object_Dump(FROM_GC(g)); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 49ce0a97d47..272be504a68 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1181,7 +1181,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } if (print_exception_recursive(&ctx, value) < 0) { PyErr_Clear(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); } Py_XDECREF(ctx.seen); @@ -1199,14 +1199,14 @@ PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) PyObject *file; if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyObject *exc = PyErr_GetRaisedException(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); - _PyObject_Dump(exc); + PyUnstable_Object_Dump(exc); Py_DECREF(exc); return; } if (file == NULL) { - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); return; } From daafacf0053e9c329b0f96447258f628dd0bd6f1 Mon Sep 17 00:00:00 2001 From: Shamil Date: Tue, 18 Nov 2025 19:34:58 +0300 Subject: [PATCH 236/417] gh-42400: Fix buffer overflow in _Py_wrealpath() for very long paths (#141529) Co-authored-by: Victor Stinner --- .../Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst | 3 +++ Python/fileutils.c | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst diff --git a/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst new file mode 100644 index 00000000000..17dc241aef9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst @@ -0,0 +1,3 @@ +Fix buffer overflow in ``_Py_wrealpath()`` for paths exceeding ``MAXPATHLEN`` bytes +by using dynamic memory allocation instead of fixed-size buffer. +Patch by Shamil Abdulaev. diff --git a/Python/fileutils.c b/Python/fileutils.c index 93abd70a34d..0c1766b8804 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2118,7 +2118,6 @@ _Py_wrealpath(const wchar_t *path, wchar_t *resolved_path, size_t resolved_path_len) { char *cpath; - char cresolved_path[MAXPATHLEN]; wchar_t *wresolved_path; char *res; size_t r; @@ -2127,12 +2126,14 @@ _Py_wrealpath(const wchar_t *path, errno = EINVAL; return NULL; } - res = realpath(cpath, cresolved_path); + res = realpath(cpath, NULL); PyMem_RawFree(cpath); if (res == NULL) return NULL; - wresolved_path = Py_DecodeLocale(cresolved_path, &r); + wresolved_path = Py_DecodeLocale(res, &r); + free(res); + if (wresolved_path == NULL) { errno = EINVAL; return NULL; From 4cfa695c953e5dfdab99ade81cee960ddf4b106d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 18 Nov 2025 09:51:18 -0800 Subject: [PATCH 237/417] GH-141686: Break cycles created by JSONEncoder.iterencode (GH-141687) --- Lib/json/encoder.py | 30 +++++++++---------- ...-11-17-16-53-49.gh-issue-141686.V-xaoI.rst | 2 ++ 2 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 5cf6d64f3ea..4c70e8b75ed 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -264,17 +264,6 @@ def floatstr(o, allow_nan=self.allow_nan, def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - ValueError=ValueError, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - _intstr=int.__repr__, ): def _iterencode_list(lst, _current_indent_level): @@ -311,7 +300,7 @@ def _iterencode_list(lst, _current_indent_level): # Subclasses of int/float may override __repr__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. - yield buf + _intstr(value) + yield buf + int.__repr__(value) elif isinstance(value, float): # see comment above for int yield buf + _floatstr(value) @@ -374,7 +363,7 @@ def _iterencode_dict(dct, _current_indent_level): key = 'null' elif isinstance(key, int): # see comment for int/float in _make_iterencode - key = _intstr(key) + key = int.__repr__(key) elif _skipkeys: continue else: @@ -399,7 +388,7 @@ def _iterencode_dict(dct, _current_indent_level): yield 'false' elif isinstance(value, int): # see comment for int/float in _make_iterencode - yield _intstr(value) + yield int.__repr__(value) elif isinstance(value, float): # see comment for int/float in _make_iterencode yield _floatstr(value) @@ -434,7 +423,7 @@ def _iterencode(o, _current_indent_level): yield 'false' elif isinstance(o, int): # see comment for int/float in _make_iterencode - yield _intstr(o) + yield int.__repr__(o) elif isinstance(o, float): # see comment for int/float in _make_iterencode yield _floatstr(o) @@ -458,4 +447,13 @@ def _iterencode(o, _current_indent_level): raise if markers is not None: del markers[markerid] - return _iterencode + + def _iterencode_once(o, _current_indent_level): + nonlocal _iterencode, _iterencode_dict, _iterencode_list + try: + yield from _iterencode(o, _current_indent_level) + finally: + # Break reference cycles due to mutually recursive closures: + del _iterencode, _iterencode_dict, _iterencode_list + + return _iterencode_once diff --git a/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst new file mode 100644 index 00000000000..87e9cb8d69b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst @@ -0,0 +1,2 @@ +Break reference cycles created by each call to :func:`json.dump` or +:meth:`json.JSONEncoder.iterencode`. From 17636ba48ce535fc1b1926c0bab26339da50631a Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Nov 2025 06:39:21 +0800 Subject: [PATCH 238/417] gh-141691: Apply ruff rules to Apple folder. (#141694) Add ruff configuration to run over the Apple build tooling and testbed script. --- .pre-commit-config.yaml | 8 ++ Apple/.ruff.toml | 22 ++++++ Apple/__main__.py | 154 +++++++++++++++++++------------------- Apple/testbed/__main__.py | 33 ++++---- 4 files changed, 126 insertions(+), 91 deletions(-) create mode 100644 Apple/.ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0311f05279..c5767ee841e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.13.2 hooks: + - id: ruff-check + name: Run Ruff (lint) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple/ - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] @@ -30,6 +34,10 @@ repos: name: Run Ruff (lint) on Tools/wasm/ args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple - id: ruff-format name: Run Ruff (format) on Doc/ args: [--check] diff --git a/Apple/.ruff.toml b/Apple/.ruff.toml new file mode 100644 index 00000000000..4cdc39ebee4 --- /dev/null +++ b/Apple/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Apple/__main__.py b/Apple/__main__.py index e76fc351798..1c588c23d6b 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -46,13 +46,12 @@ import sys import sysconfig import time -from collections.abc import Sequence +from collections.abc import Callable, Sequence from contextlib import contextmanager from datetime import datetime, timezone from os.path import basename, relpath from pathlib import Path from subprocess import CalledProcessError -from typing import Callable EnvironmentT = dict[str, str] ArgsT = Sequence[str | Path] @@ -140,17 +139,15 @@ def print_env(env: EnvironmentT) -> None: def apple_env(host: str) -> EnvironmentT: """Construct an Apple development environment for the given host.""" env = { - "PATH": ":".join( - [ - str(PYTHON_DIR / "Apple/iOS/Resources/bin"), - str(subdir(host) / "prefix"), - "/usr/bin", - "/bin", - "/usr/sbin", - "/sbin", - "/Library/Apple/usr/bin", - ] - ), + "PATH": ":".join([ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ]), } return env @@ -196,14 +193,10 @@ def clean(context: argparse.Namespace, target: str = "all") -> None: paths.append(target) if target in {"all", "hosts", "test"}: - paths.extend( - [ - path.name - for path in CROSS_BUILD_DIR.glob( - f"{context.platform}-testbed.*" - ) - ] - ) + paths.extend([ + path.name + for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*") + ]) for path in paths: delete_path(path) @@ -352,18 +345,16 @@ def download(url: str, target_dir: Path) -> Path: out_path = target_path / basename(url) if not Path(out_path).is_file(): - run( - [ - "curl", - "-Lf", - "--retry", - "5", - "--retry-all-errors", - "-o", - out_path, - url, - ] - ) + run([ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ]) else: print(f"Using cached version of {basename(url)}") return out_path @@ -468,8 +459,7 @@ def package_version(prefix_path: Path) -> str: def lib_platform_files(dirname, names): - """A file filter that ignores platform-specific files in the lib directory. - """ + """A file filter that ignores platform-specific files in lib.""" path = Path(dirname) if ( path.parts[-3] == "lib" @@ -478,7 +468,7 @@ def lib_platform_files(dirname, names): ): return names elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - ignored_names = set( + ignored_names = { name for name in names if ( @@ -486,7 +476,7 @@ def lib_platform_files(dirname, names): or name.startswith("_sysconfig_vars_") or name == "build-details.json" ) - ) + } else: ignored_names = set() @@ -499,7 +489,9 @@ def lib_non_platform_files(dirname, names): """ path = Path(dirname) if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - return set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + return ( + set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + ) else: return set() @@ -514,7 +506,8 @@ def create_xcframework(platform: str) -> str: package_path.mkdir() except FileExistsError: raise RuntimeError( - f"{platform} XCframework already exists; do you need to run with --clean?" + f"{platform} XCframework already exists; do you need to run " + "with --clean?" ) from None frameworks = [] @@ -607,7 +600,7 @@ def create_xcframework(platform: str) -> str: print(f" - {slice_name} binaries") shutil.copytree(first_path / "bin", slice_path / "bin") - # Copy the include path (this will be a symlink to the framework headers) + # Copy the include path (a symlink to the framework headers) print(f" - {slice_name} include files") shutil.copytree( first_path / "include", @@ -659,7 +652,8 @@ def create_xcframework(platform: str) -> str: # statically link those libraries into a Framework, you become # responsible for providing a privacy manifest for that framework. xcprivacy_file = { - "OpenSSL": subdir(host_triple) / "prefix/share/OpenSSL.xcprivacy" + "OpenSSL": subdir(host_triple) + / "prefix/share/OpenSSL.xcprivacy" } print(f" - {multiarch} xcprivacy files") for module, lib in [ @@ -669,7 +663,8 @@ def create_xcframework(platform: str) -> str: shutil.copy( xcprivacy_file[lib], slice_path - / f"lib-{arch}/python{version_tag}/lib-dynload/{module}.xcprivacy", + / f"lib-{arch}/python{version_tag}" + / f"lib-dynload/{module}.xcprivacy", ) print(" - build tools") @@ -692,18 +687,16 @@ def package(context: argparse.Namespace) -> None: # Clone testbed print() - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - CROSS_BUILD_DIR / context.platform / "Python.xcframework", - CROSS_BUILD_DIR / context.platform / "testbed", - ] - ) + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ]) # Build the final archive archive_name = ( @@ -757,7 +750,7 @@ def build(context: argparse.Namespace, host: str | None = None) -> None: package(context) -def test(context: argparse.Namespace, host: str | None = None) -> None: +def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028 """The implementation of the "test" command.""" if host is None: host = context.host @@ -795,18 +788,16 @@ def test(context: argparse.Namespace, host: str | None = None) -> None: / f"Frameworks/{apple_multiarch(host)}" ) - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - framework_path, - testbed_dir, - ] - ) + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ]) run( [ @@ -840,7 +831,7 @@ def apple_sim_host(platform_name: str) -> str: """Determine the native simulator target for this platform.""" for _, slice_parts in HOSTS[platform_name].items(): for host_triple in slice_parts: - parts = host_triple.split('-') + parts = host_triple.split("-") if parts[0] == platform.machine() and parts[-1] == "simulator": return host_triple @@ -968,20 +959,29 @@ def parse_args() -> argparse.Namespace: cmd.add_argument( "--simulator", help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " - "the most recently released 'entry level' iPhone device. Device " - "architecture and OS version can also be specified; e.g., " - "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " - "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + "The name of the simulator to use (eg: 'iPhone 16e'). " + "Defaults to the most recently released 'entry level' " + "iPhone device. Device architecture and OS version can also " + "be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would " + "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0." ), ) group = cmd.add_mutually_exclusive_group() group.add_argument( - "--fast-ci", action="store_const", dest="ci_mode", const="fast", - help="Add test arguments for GitHub Actions") + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + help="Add test arguments for GitHub Actions", + ) group.add_argument( - "--slow-ci", action="store_const", dest="ci_mode", const="slow", - help="Add test arguments for buildbots") + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Add test arguments for buildbots", + ) for subcommand in [configure_build, configure_host, build, ci]: subcommand.add_argument( diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 49974cb1428..0dd77ab8b82 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -32,15 +32,15 @@ def select_simulator_device(platform): json_data = json.loads(raw_json) if platform == "iOS": - # Any iOS device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". + # Any iOS device will do; we'll look for "SE" devices - but the name + # isn't consistent over time. Older Xcode versions will use "iPhone SE + # (Nth generation)"; As of 2025, they've started using "iPhone 16e". # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. + # When Xcode is updated after a new release, new devices will be + # available and old ones will be dropped from the set available on the + # latest iOS version. Select the one with the highest minimum runtime + # version - this is an indicator of the "newest" released device, which + # should always be supported on the "most recent" iOS version. se_simulators = sorted( (devicetype["minRuntimeVersion"], devicetype["name"]) for devicetype in json_data["devicetypes"] @@ -295,7 +295,8 @@ def main(): parser = argparse.ArgumentParser( description=( - "Manages the process of testing an Apple Python project through Xcode." + "Manages the process of testing an Apple Python project " + "through Xcode." ), ) @@ -336,7 +337,10 @@ def main(): run = subcommands.add_parser( "run", - usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", + usage=( + "%(prog)s [-h] [--simulator SIMULATOR] -- " + " [ ...]" + ), description=( "Run a testbed project. The arguments provided after `--` will be " "passed to the running iOS process as if they were arguments to " @@ -397,9 +401,9 @@ def main(): / "bin" ).is_dir(): print( - f"Testbed does not contain a compiled Python framework. Use " - f"`python {sys.argv[0]} clone ...` to create a runnable " - f"clone of this testbed." + "Testbed does not contain a compiled Python framework. " + f"Use `python {sys.argv[0]} clone ...` to create a " + "runnable clone of this testbed." ) sys.exit(20) @@ -411,7 +415,8 @@ def main(): ) else: print( - f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)" + "Must specify test arguments " + f"(e.g., {sys.argv[0]} run -- test)" ) print() parser.print_help(sys.stderr) From 652c764a59913327b28b32016405696a620d969e Mon Sep 17 00:00:00 2001 From: Thierry Martos <81799048+ThierryMT@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:01:09 -0800 Subject: [PATCH 239/417] gh-140381: Increase slow_fibonacci call frequency in test_profiling (#140673) --- .../test_profiling/test_sampling_profiler/test_integration.py | 4 ++-- .../next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index 4fb2c595bbe..e1c80fa6d5d 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -414,8 +414,8 @@ def main_loop(): if iteration % 3 == 0: # Very CPU intensive result = cpu_intensive_work() - elif iteration % 5 == 0: - # Expensive recursive operation + elif iteration % 2 == 0: + # Expensive recursive operation (increased frequency for slower machines) result = slow_fibonacci(12) else: # Medium operation diff --git a/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst new file mode 100644 index 00000000000..568a2b65d7d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst @@ -0,0 +1 @@ +Fix flaky test_profiling tests on i686 and s390x architectures by increasing slow_fibonacci call frequency from every 5th iteration to every 2nd iteration. From ce791541769a41beabec0f515cd62e504d46ff1c Mon Sep 17 00:00:00 2001 From: Edward Xu Date: Wed, 19 Nov 2025 08:57:59 +0800 Subject: [PATCH 240/417] gh-139103: fix free-threading `dataclass.__init__` perf issue (gh-141596) The dataclasses `__init__` function is generated dynamically by a call to `exec()` and so doesn't have deferred reference counting enabled. Enable deferred reference counting on functions when assigned as an attribute to type objects to avoid reference count contention when creating dataclass instances. --- .../2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst | 1 + Objects/typeobject.c | 12 ++++++++++++ Tools/ftscalingbench/ftscalingbench.py | 12 ++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst new file mode 100644 index 00000000000..c038dc742cc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst @@ -0,0 +1 @@ +Improve multithreaded scaling of dataclasses on the free-threaded build. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 61bcc21ce13..c99c6b3f637 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6546,6 +6546,18 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); +#ifdef Py_GIL_DISABLED + // gh-139103: Enable deferred refcounting for functions assigned + // to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically. + if (value != NULL && + PyFunction_Check(value) && + !_PyObject_HasDeferredRefcount(value)) + { + PyUnstable_Object_EnableDeferredRefcount(value); + } +#endif + PyObject *old_value = NULL; PyObject *descr = _PyType_LookupRef(metatype, name); if (descr != NULL) { diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 1a59e25189d..097a065f368 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -27,6 +27,7 @@ import sys import threading import time +from dataclasses import dataclass from operator import methodcaller # The iterations in individual benchmarks are scaled by this factor. @@ -202,6 +203,17 @@ def method_caller(): for i in range(1000 * WORK_SCALE): mc(obj) +@dataclass +class MyDataClass: + x: int + y: int + z: int + +@register_benchmark +def instantiate_dataclass(): + for _ in range(1000 * WORK_SCALE): + obj = MyDataClass(x=1, y=2, z=3) + def bench_one_thread(func): t0 = time.perf_counter_ns() func() From 7b0b70867586ef7109de60ccce94d13164dbb776 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Nov 2025 09:48:51 +0800 Subject: [PATCH 241/417] gh-141692: Add a slice-specific lib folder to iOS XCframeworks. (#141693) Modifies the iOS XCframework to include a lib folder for each slice that contains a symlinked version of the libPython dynamic library. --- Apple/__main__.py | 14 ++++++++++++++ Apple/testbed/Python.xcframework/build/utils.sh | 3 ++- Makefile.pre.in | 3 +++ .../2025-11-18-13-55-47.gh-issue-141692.tud9if.rst | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst diff --git a/Apple/__main__.py b/Apple/__main__.py index 1c588c23d6b..256966e76c2 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -477,6 +477,12 @@ def lib_platform_files(dirname, names): or name == "build-details.json" ) } + elif path.parts[-1] == "lib": + ignored_names = { + name + for name in names + if name.startswith("libpython") and name.endswith(".dylib") + } else: ignored_names = set() @@ -614,6 +620,12 @@ def create_xcframework(platform: str) -> str: slice_framework / "Headers/pyconfig.h", ) + print(f" - {slice_name} shared library") + # Create a simlink for the fat library + shared_lib = slice_path / f"lib/libpython{version_tag}.dylib" + shared_lib.parent.mkdir() + shared_lib.symlink_to("../Python.framework/Python") + print(f" - {slice_name} architecture-specific files") for host_triple, multiarch in slice_parts.items(): print(f" - {multiarch} standard library") @@ -625,6 +637,7 @@ def create_xcframework(platform: str) -> str: framework_path(host_triple, multiarch) / "lib", package_path / "Python.xcframework/lib", ignore=lib_platform_files, + symlinks=True, ) has_common_stdlib = True @@ -632,6 +645,7 @@ def create_xcframework(platform: str) -> str: framework_path(host_triple, multiarch) / "lib", slice_path / f"lib-{arch}", ignore=lib_non_platform_files, + symlinks=True, ) # Copy the host's pyconfig.h to an architecture-specific name. diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh index 961c46d014b..e7155d8b30e 100755 --- a/Apple/testbed/Python.xcframework/build/utils.sh +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -46,7 +46,8 @@ install_stdlib() { rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" else - rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + # A single-arch framework will have a libpython symlink; that can't be included at runtime + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' fi } diff --git a/Makefile.pre.in b/Makefile.pre.in index 59c3c808794..13108b1baf9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3050,6 +3050,9 @@ frameworkinstallunversionedstructure: $(LDLIBRARY) $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBDIR) + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib" + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib" $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst new file mode 100644 index 00000000000..d85c54db364 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst @@ -0,0 +1,3 @@ +Each slice of an iOS XCframework now contains a ``lib`` folder that contains +a symlink to the libpython dylib. This allows binary modules to be compiled +for iOS using dynamic libreary linking, rather than Framework linking. From 92c5de73b8d7526326c865b1a669b868f0d40c1e Mon Sep 17 00:00:00 2001 From: Ayappan Perumal Date: Wed, 19 Nov 2025 13:07:09 +0530 Subject: [PATCH 242/417] gh-141659: Fix bad file descriptor error in subprocess on AIX (GH-141660) /proc/self does not exist on AIX. --- .../2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst | 1 + Modules/_posixsubprocess.c | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst new file mode 100644 index 00000000000..eeb055c6012 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst @@ -0,0 +1 @@ +Fix bad file descriptor errors from ``_posixsubprocess`` on AIX. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 0219a3360fd..6f0a6d1d4e3 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -514,7 +514,13 @@ _close_open_fds_maybe_unsafe(int start_fd, int *fds_to_keep, proc_fd_dir = NULL; else #endif +#if defined(_AIX) + char fd_path[PATH_MAX]; + snprintf(fd_path, sizeof(fd_path), "/proc/%ld/fd", (long)getpid()); + proc_fd_dir = opendir(fd_path); +#else proc_fd_dir = opendir(FD_DIR); +#endif if (!proc_fd_dir) { /* No way to get a list of open fds. */ _close_range_except(start_fd, -1, fds_to_keep, fds_to_keep_len, From dbbf4b2e21d4aa73f54361ecda12187bacd7f6d3 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:42:16 +0200 Subject: [PATCH 243/417] Post 3.15.0a2 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 899c892631f..804aa1a0427 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.15.0a2" +#define PY_VERSION "3.15.0a2+" /*--end constants--*/ From c25a070759952b13f97ecc37ca2991c2669aee47 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 19 Nov 2025 10:16:24 +0000 Subject: [PATCH 244/417] GH-139653: Only raise an exception (or fatal error) when the stack pointer is about to overflow the stack. (GH-141711) Only raises if the stack pointer is both below the limit *and* above the stack base. This prevents false positives for user-space threads, as the stack pointer will be outside those bounds if the stack has been swapped. --- Include/internal/pycore_ceval.h | 7 +++-- InternalDocs/stack_protection.md | 9 +++++- ...-11-17-14-40-45.gh-issue-139653.LzOy1M.rst | 4 +++ Python/ceval.c | 28 ++++++++++++++----- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 47c42fccdc2..2ae84be7b33 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -217,10 +217,13 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Overflow if stack pointer is between soft limit and the base of the hardware stack. + // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. + // We could have the wrong stack limits because of limited platform support, or user-space threads. #if _Py_STACK_GROWS_DOWN - return here_addr < _tstate->c_stack_soft_limit; + return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; #else - return here_addr > _tstate->c_stack_soft_limit; + return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; #endif } diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md index fa025bd930f..14802e57d09 100644 --- a/InternalDocs/stack_protection.md +++ b/InternalDocs/stack_protection.md @@ -38,12 +38,19 @@ # Stack Protection ```python kb_used = (stack_top - stack_pointer)>>10 -if stack_pointer < hard_limit: +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") elif stack_pointer < soft_limit: raise RecursionError(f"Stack overflow (used {kb_used} kB)") ``` +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + ### Diagnosing and fixing stack overflows For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst new file mode 100644 index 00000000000..c3ae0e8adab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst @@ -0,0 +1,4 @@ +Only raise a ``RecursionError`` or trigger a fatal error if the stack +pointer is both below the limit pointer *and* above the stack base. If +outside of these bounds assume that it is OK. This prevents false positives +when user-space threads swap stacks. diff --git a/Python/ceval.c b/Python/ceval.c index 14fef42ea96..5381cd826df 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -362,9 +362,11 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) _Py_InitializeRecursionLimits(tstate); } #if _Py_STACK_GROWS_DOWN - return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; + return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; #else - return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES; + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; #endif } @@ -455,7 +457,7 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif static void -hardware_stack_limits(uintptr_t *base, uintptr_t *top) +hardware_stack_limits(uintptr_t *base, uintptr_t *top, uintptr_t sp) { #ifdef WIN32 ULONG_PTR low, high; @@ -491,10 +493,19 @@ hardware_stack_limits(uintptr_t *base, uintptr_t *top) return; } # endif - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - uintptr_t top_addr = _Py_SIZE_ROUND_UP(here_addr, 4096); + // Add some space for caller function then round to minimum page size + // This is a guess at the top of the stack, but should be a reasonably + // good guess if called from _PyThreadState_Attach when creating a thread. + // If the thread is attached deep in a call stack, then the guess will be poor. +#if _Py_STACK_GROWS_DOWN + uintptr_t top_addr = _Py_SIZE_ROUND_UP(sp + 8*sizeof(void*), SYSTEM_PAGE_SIZE); *top = top_addr; *base = top_addr - Py_C_STACK_SIZE; +# else + uintptr_t base_addr = _Py_SIZE_ROUND_DOWN(sp - 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *base = base_addr; + *top = base_addr + Py_C_STACK_SIZE; +#endif #endif } @@ -543,7 +554,8 @@ void _Py_InitializeRecursionLimits(PyThreadState *tstate) { uintptr_t base, top; - hardware_stack_limits(&base, &top); + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + hardware_stack_limits(&base, &top, here_addr); assert(top != 0); tstate_set_stack(tstate, base, top); @@ -587,7 +599,7 @@ PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the recursion_depth reaches recursion_limit. */ + if the stack pointer is between the stack base and c_stack_hard_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -596,10 +608,12 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); #if _Py_STACK_GROWS_DOWN + assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; #else + assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); if (here_addr > _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; From 52f70a6f60254fec5297d1ff731b6c1ebc52ec24 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Wed, 19 Nov 2025 05:30:53 -0500 Subject: [PATCH 245/417] Correct class name from PullDom to PullDOM (#141207) --- Doc/library/xml.dom.pulldom.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 8bceeecd463..a21cfaa4645 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -74,7 +74,7 @@ given point) or to make use of the :func:`DOMEventStream.expandNode` method and switch to DOM-related processing. -.. class:: PullDom(documentFactory=None) +.. class:: PullDOM(documentFactory=None) Subclass of :class:`xml.sax.handler.ContentHandler`. From afa0badcc587ea7500e2b4dd2ea269f8bbda5fb2 Mon Sep 17 00:00:00 2001 From: da-woods Date: Wed, 19 Nov 2025 11:38:10 +0000 Subject: [PATCH 246/417] gh-141726: Add PyDict_SetDefaultRef() to the Stable ABI (#141727) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/data/stable_abi.dat | 1 + Include/cpython/dictobject.h | 10 ---------- Include/dictobject.h | 12 ++++++++++++ Lib/test/test_stable_abi_ctypes.py | 1 + .../2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst | 1 + Misc/stable_abi.toml | 2 ++ PC/python3dll.c | 1 + 7 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 5cbf3771950..95e032655cf 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -160,6 +160,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fc..5f2f7b6d4f5 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -39,16 +39,6 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab69..0384e3131dc 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 7167646ecc6..bc834f5a681 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -165,6 +165,7 @@ SYMBOL_NAMES = ( "PyDict_MergeFromSeq2", "PyDict_New", "PyDict_Next", + "PyDict_SetDefaultRef", "PyDict_SetItem", "PyDict_SetItemString", "PyDict_Size", diff --git a/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst new file mode 100644 index 00000000000..3fdad5c6b3e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst @@ -0,0 +1 @@ +Add :c:func:`PyDict_SetDefaultRef` to the Stable ABI. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 7ee6cf1dae5..5c503f81d32 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2639,3 +2639,5 @@ added = '3.15' [const.Py_mod_token] added = '3.15' +[function.PyDict_SetDefaultRef] + added = '3.15' diff --git a/PC/python3dll.c b/PC/python3dll.c index 99e0f05fe03..35db1a660a7 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -191,6 +191,7 @@ EXPORT_FUNC(PyDict_Merge) EXPORT_FUNC(PyDict_MergeFromSeq2) EXPORT_FUNC(PyDict_New) EXPORT_FUNC(PyDict_Next) +EXPORT_FUNC(PyDict_SetDefaultRef) EXPORT_FUNC(PyDict_SetItem) EXPORT_FUNC(PyDict_SetItemString) EXPORT_FUNC(PyDict_Size) From 95296a9d40aa2d58502a09e86e2a93c03df23366 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 19 Nov 2025 13:55:10 +0200 Subject: [PATCH 247/417] gh-140875: Fix handling of unclosed charrefs before EOF in HTMLParser (GH-140904) --- Lib/html/parser.py | 29 +++-- Lib/test/test_htmlparser.py | 110 ++++++++++++++---- ...-11-02-10-44-23.gh-issue-140875.wt6B37.rst | 3 + 3 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-10-44-23.gh-issue-140875.wt6B37.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index e50620de800..80fb8c3f929 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -24,6 +24,7 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])') attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') @@ -304,10 +305,20 @@ def goahead(self, end): k = k - 1 i = self.updatepos(i, k) continue + match = incomplete_charref.match(rawdata, i) + if match: + if end: + self.handle_charref(rawdata[i+2:]) + i = self.updatepos(i, n) + break + # incomplete + break + elif i + 3 < n: # larger than "&#x" + # not the end of the buffer, and can't be confused + # with some other construct + self.handle_data("&#") + i = self.updatepos(i, i + 2) else: - if ";" in rawdata[i:]: # bail by consuming &# - self.handle_data(rawdata[i:i+2]) - i = self.updatepos(i, i+2) break elif startswith('&', i): match = entityref.match(rawdata, i) @@ -321,15 +332,13 @@ def goahead(self, end): continue match = incomplete.match(rawdata, i) if match: - # match.group() will contain at least 2 chars - if end and match.group() == rawdata[i:]: - k = match.end() - if k <= i: - k = n - i = self.updatepos(i, i + 1) + if end: + self.handle_entityref(rawdata[i+1:]) + i = self.updatepos(i, n) + break # incomplete break - elif (i + 1) < n: + elif i + 1 < n: # not the end of the buffer, and can't be confused # with some other construct self.handle_data("&") diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 19dde9362a4..e4eff1ea17a 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -109,12 +109,13 @@ def get_events(self): class TestCaseBase(unittest.TestCase): - def get_collector(self): - return EventCollector(convert_charrefs=False) + def get_collector(self, convert_charrefs=False): + return EventCollector(convert_charrefs=convert_charrefs) - def _run_check(self, source, expected_events, collector=None): + def _run_check(self, source, expected_events, + *, collector=None, convert_charrefs=False): if collector is None: - collector = self.get_collector() + collector = self.get_collector(convert_charrefs=convert_charrefs) parser = collector for s in source: parser.feed(s) @@ -128,7 +129,7 @@ def _run_check(self, source, expected_events, collector=None): def _run_check_extra(self, source, events): self._run_check(source, events, - EventCollectorExtra(convert_charrefs=False)) + collector=EventCollectorExtra(convert_charrefs=False)) class HTMLParserTestCase(TestCaseBase): @@ -187,10 +188,87 @@ def test_malformatted_charref(self): ]) def test_unclosed_entityref(self): - self._run_check("&entityref foo", [ - ("entityref", "entityref"), - ("data", " foo"), - ]) + self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('> <', [('data', '> <')], convert_charrefs=True) + + self._run_check('&undefined <', + [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('&undefined <', [('data', '&undefined <')], + convert_charrefs=True) + + self._run_check('>undefined <', + [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('>undefined <', [('data', '>undefined <')], + convert_charrefs=True) + + self._run_check('& <', [('data', '& '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('& <', [('data', '& <')], convert_charrefs=True) + + def test_eof_in_entityref(self): + self._run_check('>', [('entityref', 'gt')], convert_charrefs=False) + self._run_check('>', [('data', '>')], convert_charrefs=True) + + self._run_check('&g', [('entityref', 'g')], convert_charrefs=False) + self._run_check('&g', [('data', '&g')], convert_charrefs=True) + + self._run_check('&undefined', [('entityref', 'undefined')], + convert_charrefs=False) + self._run_check('&undefined', [('data', '&undefined')], + convert_charrefs=True) + + self._run_check('>undefined', [('entityref', 'gtundefined')], + convert_charrefs=False) + self._run_check('>undefined', [('data', '>undefined')], + convert_charrefs=True) + + self._run_check('&', [('data', '&')], convert_charrefs=False) + self._run_check('&', [('data', '&')], convert_charrefs=True) + + def test_unclosed_charref(self): + self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('{ <', [('data', '{ <')], convert_charrefs=True) + self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('« <', [('data', '\xab <')], convert_charrefs=True) + + self._run_check('� <', + [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + self._run_check('� <', + [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + + self._run_check('&# <', [('data', '&# '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&# <', [('data', '&# <')], convert_charrefs=True) + self._run_check('&#x <', [('data', '&#x '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&#x <', [('data', '&#x <')], convert_charrefs=True) + + def test_eof_in_charref(self): + self._run_check('{', [('charref', '123')], convert_charrefs=False) + self._run_check('{', [('data', '{')], convert_charrefs=True) + self._run_check('«', [('charref', 'xab')], convert_charrefs=False) + self._run_check('«', [('data', '\xab')], convert_charrefs=True) + + self._run_check('�', [('charref', '123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + self._run_check('�', [('charref', 'x123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + + self._run_check('&#', [('data', '&#')], convert_charrefs=False) + self._run_check('&#', [('data', '&#')], convert_charrefs=True) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=False) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=True) def test_bad_nesting(self): # Strangely, this *is* supposed to test that overlapping @@ -762,20 +840,6 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these - data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), - ] - for html, expected in data: - self._run_check(html, expected) - def test_eof_in_comments(self): data = [ (' + +
+
🥇
diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index f3fa441a35f..bcc24319aab 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -31,6 +31,7 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads + self.mode = mode # Store mode for later use if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, @@ -117,7 +118,7 @@ def sample(self, collector, duration_sec=10): # Pass stats to flamegraph collector if it's the right type if hasattr(collector, 'set_stats'): - collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate) + collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, mode=self.mode) expected_samples = int(duration_sec / sample_interval_sec) if num_samples < expected_samples and not is_live_mode: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 51d13a648bf..9028a8bebb1 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -62,17 +62,65 @@ def __init__(self, *args, **kwargs): self.stats = {} self._root = {"samples": 0, "children": {}, "threads": set()} self._total_samples = 0 + self._sample_count = 0 # Track actual number of samples (not thread traces) self._func_intern = {} self._string_table = StringTable() self._all_threads = set() - def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None): + # Thread status statistics (similar to LiveStatsCollector) + self.thread_status_counts = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + } + self.samples_with_gc_frames = 0 + + # Per-thread statistics + self.per_thread_stats = {} # {thread_id: {has_gil, on_cpu, gil_requested, unknown, total, gc_samples}} + + def collect(self, stack_frames, skip_idle=False): + """Override to track thread status statistics before processing frames.""" + # Increment sample count once per sample + self._sample_count += 1 + + # Collect both aggregate and per-thread statistics using base method + status_counts, has_gc_frame, per_thread_stats = self._collect_thread_status_stats(stack_frames) + + # Merge aggregate status counts + for key in status_counts: + self.thread_status_counts[key] += status_counts[key] + + # Update aggregate GC frame count + if has_gc_frame: + self.samples_with_gc_frames += 1 + + # Merge per-thread statistics + for thread_id, stats in per_thread_stats.items(): + if thread_id not in self.per_thread_stats: + self.per_thread_stats[thread_id] = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + "gc_samples": 0, + } + for key, value in stats.items(): + self.per_thread_stats[thread_id][key] += value + + # Call parent collect to process frames + super().collect(stack_frames, skip_idle=skip_idle) + + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, mode=None): """Set profiling statistics to include in flamegraph data.""" self.stats = { "sample_interval_usec": sample_interval_usec, "duration_sec": duration_sec, "sample_rate": sample_rate, - "error_rate": error_rate + "error_rate": error_rate, + "mode": mode } def export(self, filename): @@ -117,7 +165,6 @@ def _format_function_name(func): return f"{funcname} ({filename}:{lineno})" def _convert_to_flamegraph_format(self): - """Convert aggregated trie to d3-flamegraph format with string table optimization.""" if self._total_samples == 0: return { "name": self._string_table.intern("No Data"), @@ -178,6 +225,29 @@ def convert_children(children, min_samples): "strings": self._string_table.get_strings() } + # Calculate thread status percentages for display + total_threads = max(1, self.thread_status_counts["total"]) + thread_stats = { + "has_gil_pct": (self.thread_status_counts["has_gil"] / total_threads) * 100, + "on_cpu_pct": (self.thread_status_counts["on_cpu"] / total_threads) * 100, + "gil_requested_pct": (self.thread_status_counts["gil_requested"] / total_threads) * 100, + "gc_pct": (self.samples_with_gc_frames / max(1, self._sample_count)) * 100, + **self.thread_status_counts + } + + # Calculate per-thread statistics with percentages + per_thread_stats_with_pct = {} + total_samples_denominator = max(1, self._sample_count) + for thread_id, stats in self.per_thread_stats.items(): + total = max(1, stats["total"]) + per_thread_stats_with_pct[thread_id] = { + "has_gil_pct": (stats["has_gil"] / total) * 100, + "on_cpu_pct": (stats["on_cpu"] / total) * 100, + "gil_requested_pct": (stats["gil_requested"] / total) * 100, + "gc_pct": (stats["gc_samples"] / total_samples_denominator) * 100, + **stats + } + # If we only have one root child, make it the root to avoid redundant level if len(root_children) == 1: main_child = root_children[0] @@ -185,7 +255,11 @@ def convert_children(children, min_samples): old_name = self._string_table.get_string(main_child["name"]) new_name = f"Program Root: {old_name}" main_child["name"] = self._string_table.intern(new_name) - main_child["stats"] = self.stats + main_child["stats"] = { + **self.stats, + "thread_stats": thread_stats, + "per_thread_stats": per_thread_stats_with_pct + } main_child["threads"] = sorted(list(self._all_threads)) main_child["strings"] = self._string_table.get_strings() return main_child @@ -194,7 +268,11 @@ def convert_children(children, min_samples): "name": self._string_table.intern("Program Root"), "value": total_samples, "children": root_children, - "stats": self.stats, + "stats": { + **self.stats, + "thread_stats": thread_stats, + "per_thread_stats": per_thread_stats_with_pct + }, "threads": sorted(list(self._all_threads)), "strings": self._string_table.get_strings() } diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index a592f16b367..38665f5a591 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -14,6 +14,15 @@ FlamegraphCollector, ) from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.constants import ( + PROFILING_MODE_WALL, + PROFILING_MODE_CPU, + ) + from _remote_debugging import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_GIL_REQUESTED, + ) except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -657,17 +666,6 @@ def test_gecko_collector_export(self): def test_gecko_collector_markers(self): """Test Gecko profile markers for GIL and CPU state tracking.""" - try: - from _remote_debugging import ( - THREAD_STATUS_HAS_GIL, - THREAD_STATUS_ON_CPU, - THREAD_STATUS_GIL_REQUESTED, - ) - except ImportError: - THREAD_STATUS_HAS_GIL = 1 << 0 - THREAD_STATUS_ON_CPU = 1 << 1 - THREAD_STATUS_GIL_REQUESTED = 1 << 3 - collector = GeckoCollector(1000) # Status combinations for different thread states @@ -894,3 +892,312 @@ def test_pstats_collector_export(self): self.assertEqual(func1_stats[1], 2) # nc (non-recursive calls) self.assertEqual(func1_stats[2], 2.0) # tt (total time) self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + def test_flamegraph_collector_stats_accumulation(self): + """Test that FlamegraphCollector accumulates stats across samples.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # First sample + stack_frames_1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_1) + self.assertEqual(collector.thread_status_counts["has_gil"], 1) + self.assertEqual(collector.thread_status_counts["on_cpu"], 1) + self.assertEqual(collector.thread_status_counts["total"], 2) + + # Second sample + stack_frames_2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_GIL_REQUESTED), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(3, [("c.py", 3, "func_c")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_2) + + # Should accumulate + self.assertEqual(collector.thread_status_counts["has_gil"], 2) # 1 + 1 + self.assertEqual(collector.thread_status_counts["on_cpu"], 2) # 1 + 1 + self.assertEqual(collector.thread_status_counts["gil_requested"], 1) # 0 + 1 + self.assertEqual(collector.thread_status_counts["total"], 5) # 2 + 3 + + # Test GC sample tracking + stack_frames_gc = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("~", 0, "")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames_gc) + self.assertEqual(collector.samples_with_gc_frames, 1) + + # Another sample without GC + collector.collect(stack_frames_1) + self.assertEqual(collector.samples_with_gc_frames, 1) # Still 1 + + # Another GC sample + collector.collect(stack_frames_gc) + self.assertEqual(collector.samples_with_gc_frames, 2) + + def test_flamegraph_collector_per_thread_stats(self): + """Test per-thread statistics tracking in FlamegraphCollector.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Multiple threads with different states + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + MockThreadInfo(3, [("c.py", 3, "func_c")], status=THREAD_STATUS_GIL_REQUESTED), + ], + ) + ] + collector.collect(stack_frames) + + # Check per-thread stats + self.assertIn(1, collector.per_thread_stats) + self.assertIn(2, collector.per_thread_stats) + self.assertIn(3, collector.per_thread_stats) + + # Thread 1: has GIL + self.assertEqual(collector.per_thread_stats[1]["has_gil"], 1) + self.assertEqual(collector.per_thread_stats[1]["on_cpu"], 0) + self.assertEqual(collector.per_thread_stats[1]["total"], 1) + + # Thread 2: on CPU + self.assertEqual(collector.per_thread_stats[2]["has_gil"], 0) + self.assertEqual(collector.per_thread_stats[2]["on_cpu"], 1) + self.assertEqual(collector.per_thread_stats[2]["total"], 1) + + # Thread 3: waiting + self.assertEqual(collector.per_thread_stats[3]["gil_requested"], 1) + self.assertEqual(collector.per_thread_stats[3]["total"], 1) + + # Test accumulation across samples + stack_frames_2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_2) + + self.assertEqual(collector.per_thread_stats[1]["has_gil"], 1) + self.assertEqual(collector.per_thread_stats[1]["on_cpu"], 1) + self.assertEqual(collector.per_thread_stats[1]["total"], 2) + + def test_flamegraph_collector_percentage_calculations(self): + """Test that percentage calculations are correct in exported data.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Create scenario: 60% GIL held, 40% not held + for i in range(6): + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + for i in range(4): + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Export to get calculated percentages + data = collector._convert_to_flamegraph_format() + thread_stats = data["stats"]["thread_stats"] + + self.assertAlmostEqual(thread_stats["has_gil_pct"], 60.0, places=1) + self.assertAlmostEqual(thread_stats["on_cpu_pct"], 40.0, places=1) + self.assertEqual(thread_stats["total"], 10) + + def test_flamegraph_collector_mode_handling(self): + """Test that profiling mode is correctly passed through to exported data.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Collect some data + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats with mode + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_CPU + ) + + data = collector._convert_to_flamegraph_format() + self.assertEqual(data["stats"]["mode"], PROFILING_MODE_CPU) + + def test_flamegraph_collector_zero_samples_edge_case(self): + """Test that collector handles zero samples gracefully.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Export without collecting any samples + data = collector._convert_to_flamegraph_format() + + # Should return a valid structure with no data + self.assertIn("name", data) + self.assertEqual(data["value"], 0) + self.assertIn("children", data) + self.assertEqual(len(data["children"]), 0) + + def test_flamegraph_collector_json_structure_includes_stats(self): + """Test that exported JSON includes thread_stats and per_thread_stats.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Collect some data with multiple threads + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_WALL + ) + + # Export and verify structure + data = collector._convert_to_flamegraph_format() + + # Check that stats object exists and contains expected fields + self.assertIn("stats", data) + stats = data["stats"] + + # Verify thread_stats exists and has expected structure + self.assertIn("thread_stats", stats) + thread_stats = stats["thread_stats"] + self.assertIn("has_gil_pct", thread_stats) + self.assertIn("on_cpu_pct", thread_stats) + self.assertIn("gil_requested_pct", thread_stats) + self.assertIn("gc_pct", thread_stats) + self.assertIn("total", thread_stats) + + # Verify per_thread_stats exists and has data for both threads + self.assertIn("per_thread_stats", stats) + per_thread_stats = stats["per_thread_stats"] + self.assertIn(1, per_thread_stats) + self.assertIn(2, per_thread_stats) + + # Check per-thread structure + for thread_id in [1, 2]: + thread_data = per_thread_stats[thread_id] + self.assertIn("has_gil_pct", thread_data) + self.assertIn("on_cpu_pct", thread_data) + self.assertIn("gil_requested_pct", thread_data) + self.assertIn("gc_pct", thread_data) + self.assertIn("total", thread_data) + + def test_flamegraph_collector_per_thread_gc_percentage(self): + """Test that per-thread GC percentage uses total samples as denominator.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Create 10 samples total: + # - Thread 1 appears in all 10 samples, has GC in 2 of them + # - Thread 2 appears in only 5 samples, has GC in 1 of them + + # First 5 samples: both threads, thread 1 has GC in 2 + for i in range(5): + has_gc = i < 2 # First 2 samples have GC for thread 1 + frames_1 = [("~", 0, "")] if has_gc else [("a.py", 1, "func_a")] + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, frames_1, status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Next 5 samples: only thread 1, thread 2 appears in first of these with GC + for i in range(5): + if i == 0: + # Thread 2 appears in this sample with GC + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("~", 0, "")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + else: + # Only thread 1 + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats and export + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_WALL + ) + + data = collector._convert_to_flamegraph_format() + per_thread_stats = data["stats"]["per_thread_stats"] + + # Thread 1: appeared in 10 samples, had GC in 2 + # GC percentage should be 2/10 = 20% (using total samples, not thread appearances) + self.assertEqual(collector.per_thread_stats[1]["gc_samples"], 2) + self.assertEqual(collector.per_thread_stats[1]["total"], 10) + self.assertAlmostEqual(per_thread_stats[1]["gc_pct"], 20.0, places=1) + + # Thread 2: appeared in 6 samples, had GC in 1 + # GC percentage should be 1/10 = 10% (using total samples, not thread appearances) + self.assertEqual(collector.per_thread_stats[2]["gc_samples"], 1) + self.assertEqual(collector.per_thread_stats[2]["total"], 6) + self.assertAlmostEqual(per_thread_stats[2]["gc_pct"], 10.0, places=1) diff --git a/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst b/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst new file mode 100644 index 00000000000..5742beeb85c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst @@ -0,0 +1,5 @@ +The ``profiling.sampling`` flamegraph profiler now displays thread status +statistics showing the percentage of time threads spend holding the GIL, +running without the GIL, waiting for the GIL, and performing garbage +collection. These statistics help identify GIL contention and thread behavior +patterns. When filtering by thread, the display shows per-thread metrics. From 056d6c5ed90bfed2861098f1e42640d6ea62cac8 Mon Sep 17 00:00:00 2001 From: yihong Date: Sun, 30 Nov 2025 10:49:13 +0800 Subject: [PATCH 364/417] gh-141999: Handle KeyboardInterrupt when sampling in the new tachyon profiler (#142000) --- Lib/profiling/sampling/sample.py | 80 ++++++++++--------- .../test_sampling_profiler/test_profiler.py | 40 ++++++++++ ...-11-27-11-39-50.gh-issue-141999._FKGlu.rst | 2 + 3 files changed, 85 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index bcc24319aab..82c0d3959ba 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -57,50 +57,56 @@ def sample(self, collector, duration_sec=10): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + interrupted = False - while running_time < duration_sec: - # Check if live collector wants to stop - if hasattr(collector, 'running') and not collector.running: - break - - current_time = time.perf_counter() - if next_time < current_time: - try: - stack_frames = self.unwinder.get_stack_trace() - collector.collect(stack_frames) - except ProcessLookupError: - duration_sec = current_time - start_time + try: + while running_time < duration_sec: + # Check if live collector wants to stop + if hasattr(collector, 'running') and not collector.running: break - except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): - collector.collect_failed_sample() - errors += 1 - except Exception as e: - if not self._is_process_running(): + + current_time = time.perf_counter() + if next_time < current_time: + try: + stack_frames = self.unwinder.get_stack_trace() + collector.collect(stack_frames) + except ProcessLookupError: + duration_sec = current_time - start_time break - raise e from None + except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + collector.collect_failed_sample() + errors += 1 + except Exception as e: + if not self._is_process_running(): + break + raise e from None - # Track actual sampling intervals for real-time stats - if num_samples > 0: - actual_interval = current_time - last_sample_time - self.sample_intervals.append( - 1.0 / actual_interval - ) # Convert to Hz - self.total_samples += 1 + # Track actual sampling intervals for real-time stats + if num_samples > 0: + actual_interval = current_time - last_sample_time + self.sample_intervals.append( + 1.0 / actual_interval + ) # Convert to Hz + self.total_samples += 1 - # Print real-time statistics if enabled - if ( - self.realtime_stats - and (current_time - last_realtime_update) - >= realtime_update_interval - ): - self._print_realtime_stats() - last_realtime_update = current_time + # Print real-time statistics if enabled + if ( + self.realtime_stats + and (current_time - last_realtime_update) + >= realtime_update_interval + ): + self._print_realtime_stats() + last_realtime_update = current_time - last_sample_time = current_time - num_samples += 1 - next_time += sample_interval_sec + last_sample_time = current_time + num_samples += 1 + next_time += sample_interval_sec + running_time = time.perf_counter() - start_time + except KeyboardInterrupt: + interrupted = True running_time = time.perf_counter() - start_time + print("Interrupted by user.") # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: @@ -121,7 +127,7 @@ def sample(self, collector, duration_sec=10): collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, mode=self.mode) expected_samples = int(duration_sec / sample_interval_sec) - if num_samples < expected_samples and not is_live_mode: + if num_samples < expected_samples and not is_live_mode and not interrupted: print( f"Warning: missed {expected_samples - num_samples} samples " f"from the expected total of {expected_samples} " diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 2d129dc8db5..822f559561e 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -224,6 +224,46 @@ def test_sample_profiler_missed_samples_warning(self): self.assertIn("Warning: missed", result) self.assertIn("samples from the expected total", result) + def test_sample_profiler_keyboard_interrupt(self): + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.side_effect = [ + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ], + KeyboardInterrupt(), + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + mock_collector = mock.MagicMock() + times = [0.0, 0.01, 0.02, 0.03, 0.04] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + try: + profiler.sample(mock_collector, duration_sec=1.0) + except KeyboardInterrupt: + self.fail( + "KeyboardInterrupt was not handled by the profiler" + ) + result = output.getvalue() + self.assertIn("Interrupted by user.", result) + self.assertIn("Captured", result) + self.assertIn("samples", result) + self.assertNotIn("Warning: missed", result) + @force_not_colorized_test_class class TestPrintSampledStats(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst b/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst new file mode 100644 index 00000000000..3b54a831b54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst @@ -0,0 +1,2 @@ +Correctly allow :exc:`KeyboardInterrupt` to stop the process when using +:mod:`!profiling.sampling`. From cd4d0ae75c0a132f4fdc68ad0d043898931ae999 Mon Sep 17 00:00:00 2001 From: Thierry Martos <81799048+ThierryMT@users.noreply.github.com> Date: Sun, 30 Nov 2025 04:47:31 -0800 Subject: [PATCH 365/417] Improve clarity in tutorial introduction (#140669) --- Doc/tutorial/introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index fb491149793..deabac52530 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -49,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. From 229ed3dd1f97b2f87629a240b90eddba5ded67bf Mon Sep 17 00:00:00 2001 From: flovent Date: Mon, 1 Dec 2025 05:10:01 +0800 Subject: [PATCH 366/417] gh-142067: Add missing default value for param in `multiprocessing.Pipe`'s doc (GH-142109) --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index cbc98b256a9..b297001f2b5 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -890,7 +890,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the From 981ce0cf3af68cd8eee41fed19969cf0f2218572 Mon Sep 17 00:00:00 2001 From: Tadej Magajna Date: Mon, 1 Dec 2025 03:14:20 +0100 Subject: [PATCH 367/417] gh-142066: Fix grammar in multiprocessing Pipes and Queues (GH-142121) docs: Fix grammar in multiprocessing Pipes and Queues (gh-142066) --- Doc/library/multiprocessing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b297001f2b5..92605c57527 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -832,8 +832,8 @@ raising an exception. One difference from other Python queue implementations, is that :mod:`multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. From 3e2c55749326809a2fc76b9f2cb87a6f89037ebe Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sun, 30 Nov 2025 18:50:05 -0800 Subject: [PATCH 368/417] gh-141473: Document not calling Popen.wait after Popen.communicate times out. (GH-142101) Document not calling Popen.wait after Popen.communicate times out. Closes #141473 --- Doc/library/subprocess.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 43da804b62b..b8dfcc31077 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -846,6 +846,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data From b708485d1ac30f793d74b4fe7121e86dd35cda79 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:16:37 +0000 Subject: [PATCH 369/417] Docs: Upgrade Sphinx to 9.0 (#142114) --- Doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements.txt b/Doc/requirements.txt index d5f7b473c3a..716772b7f28 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,7 +7,7 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # Keep this version in sync with ``Doc/conf.py``. -sphinx~=8.2.0 +sphinx~=9.0.0 blurb From d4fa70706c95a5eec4cca340c6232c92168f6cff Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:36:17 +0000 Subject: [PATCH 370/417] gh-139707: Add mechanism for distributors to supply error messages for missing stdlib modules (GH-140783) --- Doc/using/configure.rst | 24 +++++++++ Doc/whatsnew/3.15.rst | 6 +++ Lib/test/test_traceback.py | 23 ++++++++- Lib/traceback.py | 11 +++- Makefile.pre.in | 6 +++ ...-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst | 4 ++ Tools/build/check_extension_modules.py | 50 +++++++++++++++++++ configure | 18 +++++++ configure.ac | 9 ++++ 9 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index cdadbe51417..e140ca5d71f 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -322,6 +322,30 @@ General Options .. versionadded:: 3.11 +.. option:: --with-missing-stdlib-config=FILE + + Path to a `JSON `_ configuration file + containing custom error messages for missing :term:`standard library` modules. + + This option is intended for Python distributors who wish to provide + distribution-specific guidance when users encounter standard library + modules that are missing or packaged separately. + + The JSON file should map missing module names to custom error message strings. + For example, if your distribution packages :mod:`tkinter` and + :mod:`_tkinter` separately and excludes :mod:`!_gdbm` for legal reasons, + the configuration could contain: + + .. code-block:: json + + { + "_gdbm": "The '_gdbm' module is not available in this distribution" + "tkinter": "Install the python-tk package to use tkinter", + "_tkinter": "Install the python-tk package to use tkinter", + } + + .. versionadded:: next + .. option:: --enable-pystats Turn on internal Python performance statistics gathering. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 4882ddb4310..27e3f23e47c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1247,6 +1247,12 @@ Build changes set to ``no`` or with :option:`!--without-system-libmpdec`. (Contributed by Sergey B Kirpichev in :gh:`115119`.) +* The new configure option :option:`--with-missing-stdlib-config=FILE` allows + distributors to pass a `JSON `_ + configuration file containing custom error messages for :term:`standard library` + modules that are missing or packaged separately. + (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) + Porting to Python 3.15 ====================== diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index bf57867a871..3876f1a74bb 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -5051,7 +5051,7 @@ def test_no_site_package_flavour(self): b"or to enable your virtual environment?"), stderr ) - def test_missing_stdlib_package(self): + def test_missing_stdlib_module(self): code = """ import sys sys.stdlib_module_names |= {'spam'} @@ -5061,6 +5061,27 @@ def test_missing_stdlib_package(self): self.assertIn(b"Standard library module 'spam' was not found", stderr) + code = """ + import sys + import traceback + traceback._MISSING_STDLIB_MODULE_MESSAGES = {'spam': "Install 'spam4life' for 'spam'"} + sys.stdlib_module_names |= {'spam'} + import spam + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertIn(b"Install 'spam4life' for 'spam'", stderr) + + @unittest.skipIf(sys.platform == "win32", "Non-Windows test") + def test_windows_only_module_error(self): + try: + import msvcrt # noqa: F401 + except ModuleNotFoundError: + formatted = traceback.format_exc() + self.assertIn("Unsupported platform for Windows-only standard library module 'msvcrt'", formatted) + else: + self.fail("ModuleNotFoundError was not raised") + class TestColorizedTraceback(unittest.TestCase): maxDiff = None diff --git a/Lib/traceback.py b/Lib/traceback.py index 9b4b8c7d566..8a3e0f77e76 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -14,6 +14,11 @@ from contextlib import suppress +try: + from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES +except ImportError: + _MISSING_STDLIB_MODULE_MESSAGES = {} + __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', @@ -1110,7 +1115,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, elif exc_type and issubclass(exc_type, ModuleNotFoundError): module_name = getattr(exc_value, "name", None) if module_name in sys.stdlib_module_names: - self._str = f"Standard library module '{module_name}' was not found" + message = _MISSING_STDLIB_MODULE_MESSAGES.get( + module_name, + f"Standard library module {module_name!r} was not found" + ) + self._str = message elif sys.flags.no_site: self._str += (". Site initialization is disabled, did you forget to " + "add the site-packages directory to sys.path " diff --git a/Makefile.pre.in b/Makefile.pre.in index 7b8e7ec0965..816080faa1f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1604,6 +1604,11 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt # dependency on BUILDPYTHON ensures that the target is run last .PHONY: checksharedmods checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) + @if [ -n "@MISSING_STDLIB_CONFIG@" ]; then \ + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info --with-missing-stdlib-config="@MISSING_STDLIB_CONFIG@"; \ + else \ + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info; \ + fi @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py .PHONY: rundsymutil @@ -2820,6 +2825,7 @@ libinstall: all $(srcdir)/Modules/xxmodule.c $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_missing_stdlib_info.py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against diff --git a/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst b/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst new file mode 100644 index 00000000000..d9870d26704 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst @@ -0,0 +1,4 @@ +Add configure option :option:`--with-missing-stdlib-config=FILE` allows +which distributors to pass a `JSON `_ +configuration file containing custom error messages for missing +:term:`standard library` modules. diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 668db8df0bd..f23c1d5286f 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -23,9 +23,11 @@ import _imp import argparse import enum +import json import logging import os import pathlib +import pprint import re import sys import sysconfig @@ -116,6 +118,18 @@ help="Print a list of module names to stdout and exit", ) +parser.add_argument( + "--generate-missing-stdlib-info", + action="store_true", + help="Generate file with stdlib module info", +) + +parser.add_argument( + "--with-missing-stdlib-config", + metavar="CONFIG_FILE", + help="Path to JSON config file with custom missing module messages", +) + @enum.unique class ModuleState(enum.Enum): @@ -281,6 +295,39 @@ def list_module_names(self, *, all: bool = False) -> set[str]: names.update(WINDOWS_MODULES) return names + def generate_missing_stdlib_info(self, config_path: str | None = None) -> None: + config_messages = {} + if config_path: + try: + with open(config_path, encoding='utf-8') as f: + config_messages = json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + raise RuntimeError(f"Failed to load missing stdlib config {config_path!r}") from e + + messages = {} + for name in WINDOWS_MODULES: + messages[name] = f"Unsupported platform for Windows-only standard library module {name!r}" + + for modinfo in self.modules: + if modinfo.state in (ModuleState.DISABLED, ModuleState.DISABLED_SETUP): + messages[modinfo.name] = f"Standard library module disabled during build {modinfo.name!r} was not found" + elif modinfo.state == ModuleState.NA: + messages[modinfo.name] = f"Unsupported platform for standard library module {modinfo.name!r}" + + messages.update(config_messages) + + content = f'''\ +# Standard library information used by the traceback module for more informative +# ModuleNotFound error messages. +# Generated by check_extension_modules.py + +_MISSING_STDLIB_MODULE_MESSAGES = {pprint.pformat(messages)} +''' + + output_path = self.builddir / "_missing_stdlib_info.py" + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) + def get_builddir(self) -> pathlib.Path: try: with open(self.pybuilddir_txt, encoding="utf-8") as f: @@ -499,6 +546,9 @@ def main() -> None: names = checker.list_module_names(all=True) for name in sorted(names): print(name) + elif args.generate_missing_stdlib_info: + checker.check() + checker.generate_missing_stdlib_info(args.with_missing_stdlib_config) else: checker.check() checker.summary(verbose=args.verbose) diff --git a/configure b/configure index 4bcb639d781..620878bb181 100755 --- a/configure +++ b/configure @@ -1012,6 +1012,7 @@ UNIVERSALSDK host_exec_prefix host_prefix MACHDEP +MISSING_STDLIB_CONFIG PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -1083,6 +1084,7 @@ ac_user_opts=' enable_option_checking with_build_python with_pkg_config +with_missing_stdlib_config enable_universalsdk with_universal_archs with_framework_name @@ -1862,6 +1864,9 @@ Optional Packages: --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) + --with-missing-stdlib-config=FILE + File with custom module error messages for missing + stdlib modules --with-universal-archs=ARCH specify the kind of macOS universal binary that should be created. This option is only valid when @@ -4095,6 +4100,19 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then as_fn_error $? "pkg-config is required" "$LINENO" 5] fi + +# Check whether --with-missing-stdlib-config was given. +if test ${with_missing_stdlib_config+y} +then : + withval=$with_missing_stdlib_config; MISSING_STDLIB_CONFIG="$withval" +else case e in #( + e) MISSING_STDLIB_CONFIG="" + ;; +esac +fi + + + # Set name for machine-dependent library files { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 diff --git a/configure.ac b/configure.ac index a1f1cf207c5..8ef479fe320 100644 --- a/configure.ac +++ b/configure.ac @@ -307,6 +307,15 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then AC_MSG_ERROR([pkg-config is required])] fi +dnl Allow distributors to provide custom missing stdlib module error messages +AC_ARG_WITH([missing-stdlib-config], + [AS_HELP_STRING([--with-missing-stdlib-config=FILE], + [File with custom module error messages for missing stdlib modules])], + [MISSING_STDLIB_CONFIG="$withval"], + [MISSING_STDLIB_CONFIG=""] +) +AC_SUBST([MISSING_STDLIB_CONFIG]) + # Set name for machine-dependent library files AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) AC_MSG_CHECKING([MACHDEP]) From 5a4c4a033a4a54481be6870aa1896fad732555b5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 1 Dec 2025 17:26:07 +0200 Subject: [PATCH 371/417] gh-119451: Fix a potential denial of service in http.client (GH-119454) Reading the whole body of the HTTP response could cause OOM if the Content-Length value is too large even if the server does not send a large amount of data. Now the HTTP client reads large data by chunks, therefore the amount of consumed memory is proportional to the amount of sent data. --- Lib/http/client.py | 28 ++++++-- Lib/test/test_httplib.py | 66 +++++++++++++++++++ ...-05-23-11-47-48.gh-issue-119451.qkJe9-.rst | 5 ++ 3 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 4b9a61cfc11..73c3256734a 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -111,6 +111,11 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -642,10 +647,25 @@ def _safe_read(self, amt): reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - data = self.fp.read(amt) - if len(data) < amt: - raise IncompleteRead(data, amt-len(data)) - return data + cursize = min(amt, _MIN_READ_BUF_SIZE) + data = self.fp.read(cursize) + if len(data) >= amt: + return data + if len(data) < cursize: + raise IncompleteRead(data, amt - len(data)) + + data = io.BytesIO(data) + data.seek(0, 2) + while True: + # This is a geometric increase in read size (never more than + # doubling out the current length of data per loop iteration). + delta = min(cursize, amt - cursize) + data.write(self.fp.read(delta)) + if data.tell() >= amt: + return data.getvalue() + cursize += delta + if data.tell() < cursize: + raise IncompleteRead(data.getvalue(), amt - data.tell()) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 47e3914d1dd..44044d0385c 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1511,6 +1511,72 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst new file mode 100644 index 00000000000..6d6f25cd2f8 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`http.client` module. +When connecting to a malicious server, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 694922cf40aa3a28f898b5f5ee08b71b4922df70 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 1 Dec 2025 17:28:15 +0200 Subject: [PATCH 372/417] gh-119342: Fix a potential denial of service in plistlib (GH-119343) Reading a specially prepared small Plist file could cause OOM because file's read(n) preallocates a bytes object for reading the specified amount of data. Now plistlib reads large data by chunks, therefore the upper limit of consumed memory is proportional to the size of the input file. --- Lib/plistlib.py | 31 ++++++++++------ Lib/test/test_plistlib.py | 37 +++++++++++++++++-- ...-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst | 5 +++ 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 67e832db217..655c51eea3d 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -73,6 +73,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 class UID: def __init__(self, data): @@ -508,12 +511,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -573,22 +588,16 @@ def _read_object(self, ref): elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') elif tokenH == 0x80: # UID diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index a0c76e5dec5..de2a2fd1fc3 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -903,8 +903,7 @@ def test_dump_naive_datetime_with_aware_datetime_option(self): class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -916,7 +915,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -1024,6 +1027,34 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + def check(data): + with open(os_helper.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(os_helper.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(os_helper.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + def test_load_aware_datetime(self): data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' diff --git a/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst new file mode 100644 index 00000000000..04fd8faca4c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`plistlib` module. +When reading a Plist file received from untrusted source, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 52f9b5f580b6b85dbf08fa23103d17a60455bc20 Mon Sep 17 00:00:00 2001 From: Yashraj Date: Mon, 1 Dec 2025 21:21:50 +0530 Subject: [PATCH 373/417] gh-141004: Document descriptor and dict proxy type objects (GH-141803) Co-authored-by: Peter Bierma Co-authored-by: Victor Stinner --- Doc/c-api/descriptor.rst | 42 ++++++++++++++++++++++++++++++++++++++++ Doc/c-api/dict.rst | 11 +++++++++++ 2 files changed, 53 insertions(+) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 22c3b790cc3..313c534545a 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -21,12 +21,46 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. + + + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. + + .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MemberDescriptorType` + objects in Python. + + .. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. + + .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) @@ -55,6 +89,14 @@ Built-in descriptors :class:`classmethod` in the Python layer. +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined in + C extension types, and is the same object as :class:`classmethod` + in Python. + + .. c:function:: PyObject *PyClassMethod_New(PyObject *callable) Create a new :class:`classmethod` object wrapping *callable*. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index ede1699cfeb..9c4428ced41 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -43,6 +43,17 @@ Dictionary Objects prevent modification of the dictionary for non-dynamic class types. +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. From f87eb4d7cd896c8f2d0df1a679bc32a4ac737fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Kiss=20Koll=C3=A1r?= Date: Mon, 1 Dec 2025 17:34:14 +0000 Subject: [PATCH 374/417] gh-138122: New Tachyon UI (#142116) Co-authored-by: Pablo Galindo Salgado --- Lib/profiling/sampling/flamegraph.css | 1744 +++++++++++------ Lib/profiling/sampling/flamegraph.js | 1091 +++++++---- .../sampling/flamegraph_template.html | 448 +++-- Lib/profiling/sampling/sample.py | 7 +- Lib/profiling/sampling/stack_collector.py | 4 +- .../test_sampling_profiler/test_collectors.py | 2 +- 6 files changed, 2129 insertions(+), 1167 deletions(-) diff --git a/Lib/profiling/sampling/flamegraph.css b/Lib/profiling/sampling/flamegraph.css index 0a6fde2ad32..1703815acd9 100644 --- a/Lib/profiling/sampling/flamegraph.css +++ b/Lib/profiling/sampling/flamegraph.css @@ -1,158 +1,1154 @@ -body { - font-family: - "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", "Geneva", - "Verdana", sans-serif; +/* ========================================================================== + Flamegraph Viewer - CSS + Python-branded profiler with dark/light theme support + ========================================================================== */ + +/* -------------------------------------------------------------------------- + CSS Variables & Theme System + -------------------------------------------------------------------------- */ + +:root { + /* Typography */ + --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", + "Geneva", "Verdana", sans-serif; + --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; + + /* Python brand colors (theme-independent) */ + --python-blue: #3776ab; + --python-blue-light: #4584bb; + --python-blue-lighter: #5592cc; + --python-gold: #ffd43b; + --python-gold-dark: #ffcd02; + --python-gold-light: #ffdc5c; + + /* Heat palette - defined per theme below */ + + /* Layout */ + --sidebar-width: 280px; + --sidebar-collapsed: 44px; + --topbar-height: 52px; + --statusbar-height: 32px; + + /* Transitions */ + --transition-fast: 0.15s ease; + --transition-normal: 0.25s ease; +} + +/* Light theme (default) - Python yellow-to-blue heat palette */ +:root, [data-theme="light"] { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #e9ecef; + --border: #e9ecef; + --border-subtle: #f0f2f5; + + --text-primary: #2e3338; + --text-secondary: #5a6c7d; + --text-muted: #8b949e; + + --accent: #3776ab; + --accent-hover: #2d5aa0; + --accent-glow: rgba(55, 118, 171, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); + + --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); + + /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ + --heat-1: #d6e9f8; + --heat-2: #a8d0ef; + --heat-3: #7ba3d1; + --heat-4: #ffe6a8; + --heat-5: #ffd43b; + --heat-6: #ffb84d; + --heat-7: #ff9966; + --heat-8: #ff6347; +} + +/* Dark theme - teal-to-orange heat palette */ +[data-theme="dark"] { + --bg-primary: #0d1117; + --bg-secondary: #161b22; + --bg-tertiary: #21262d; + --border: #30363d; + --border-subtle: #21262d; + + --text-primary: #e6edf3; + --text-secondary: #8b949e; + --text-muted: #6e7681; + + --accent: #58a6ff; + --accent-hover: #79b8ff; + --accent-glow: rgba(88, 166, 255, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); + + --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); + + /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ + --heat-1: #1e3a5f; + --heat-2: #2d5580; + --heat-3: #4a7ba7; + --heat-4: #5a9fa8; + --heat-5: #7ec488; + --heat-6: #c4de6a; + --heat-7: #f4d44d; + --heat-8: #ff6b35; +} + +/* -------------------------------------------------------------------------- + Base Styles + -------------------------------------------------------------------------- */ + +*, *::before, *::after { + box-sizing: border-box; +} + +html, body { margin: 0; padding: 0; - background: #ffffff; - color: #2e3338; + height: 100%; + overflow: hidden; +} + +body { + font-family: var(--font-sans); + font-size: 14px; line-height: 1.6; + color: var(--text-primary); + background: var(--bg-primary); + transition: background var(--transition-normal), color var(--transition-normal); } -.header { - background: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); - color: white; - padding: 32px 0; - box-shadow: 0 2px 10px rgba(55, 118, 171, 0.2); -} +/* -------------------------------------------------------------------------- + Layout Structure + -------------------------------------------------------------------------- */ -.header-content { - max-width: 1200px; - margin: 0 auto; - padding: 0 24px; +.app-layout { display: flex; flex-direction: column; - align-items: center; - justify-content: center; - text-align: center; - gap: 20px; + height: 100vh; } -.python-logo { - width: auto; - height: 72px; - margin-bottom: 12px; /* tighter spacing to avoid visual gap */ +.main-content { + display: flex; + flex: 1; + min-height: 0; +} + +/* -------------------------------------------------------------------------- + Top Bar + -------------------------------------------------------------------------- */ + +.top-bar { + height: 56px; + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; flex-shrink: 0; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); +} + +/* Brand / Logo */ +.brand { + display: flex; + align-items: center; + gap: 12px; + color: white; + text-decoration: none; + flex-shrink: 0; +} + +.brand-logo { display: flex; align-items: center; justify-content: center; + width: 28px; + height: 28px; + flex-shrink: 0; } -.python-logo img { - height: 72px; - width: auto; - display: block; /* avoid baseline alignment issues */ - vertical-align: middle; - /* subtle shadow that does not affect layout */ - filter: drop-shadow(0 2px 6px rgba(0, 0, 0, 0.1)); +/* Style the inlined SVG/img inside brand-logo */ +.brand-logo svg, +.brand-logo img { + width: 28px; + height: 28px; + display: block; + object-fit: contain; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); } -.header-text h1 { - margin: 0; - font-size: 2.5em; - font-weight: 600; - color: white; - text-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); +.brand-info { + display: flex; + flex-direction: column; + line-height: 1.15; } -.header-text .subtitle { - margin: 8px 0 0 0; - font-size: 1.1em; - color: rgba(255, 255, 255, 0.9); - font-weight: 300; -} - -.header-search { - width: 100%; - max-width: 500px; -} - -.header-search #search-input { - width: 100%; - padding: 12px 20px; - border: 2px solid rgba(255, 255, 255, 0.2); - border-radius: 25px; +.brand-text { + font-weight: 700; font-size: 16px; - font-family: inherit; - background: rgba(255, 255, 255, 0.95); + letter-spacing: -0.3px; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); +} + +.brand-subtitle { + font-weight: 500; + font-size: 10px; + opacity: 0.9; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.brand-divider { + width: 1px; + height: 16px; + background: rgba(255, 255, 255, 0.3); +} + +/* Search */ +.search-wrapper { + flex: 1; + max-width: 360px; + position: relative; +} + +.search-input { + width: 100%; + padding: 8px 36px 8px 14px; + font-family: var(--font-sans); + font-size: 13px; color: #2e3338; - transition: all 0.3s ease; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); - backdrop-filter: blur(10px); -} - -.header-search #search-input:focus { + background: rgba(255, 255, 255, 0.95); + border: 2px solid rgba(255, 255, 255, 0.3); + border-radius: 20px; outline: none; - border-color: rgba(255, 255, 255, 0.8); - background: rgba(255, 255, 255, 1); - box-shadow: 0 6px 20px rgba(0, 0, 0, 0.15); - transform: translateY(-2px); + transition: all var(--transition-fast); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); } -.header-search #search-input::placeholder { +.search-input::placeholder { color: #6c757d; } -.stats-section { - background: #ffffff; - padding: 24px 0; - border-bottom: 1px solid #e9ecef; +.search-input:focus { + border-color: rgba(255, 255, 255, 0.8); + background: white; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.15); } -.stats-container { - max-width: 1200px; - margin: 0 auto; - padding: 0 24px; - display: grid; - grid-template-columns: repeat(3, 1fr); - gap: 20px; +/* Dark theme search input */ +[data-theme="dark"] .search-input { + color: #e6edf3; + background: rgba(33, 38, 45, 0.95); + border: 2px solid rgba(88, 166, 255, 0.3); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); } -/* Compact Thread Stats Bar - Colorful Square Design */ -.thread-stats-bar { - background: rgba(255, 255, 255, 0.95); - padding: 12px 24px; +[data-theme="dark"] .search-input::placeholder { + color: #8b949e; +} + +[data-theme="dark"] .search-input:focus { + border-color: rgba(88, 166, 255, 0.6); + background: rgba(33, 38, 45, 1); + box-shadow: 0 4px 16px rgba(88, 166, 255, 0.2); +} + +.search-input.has-matches { + border-color: rgba(40, 167, 69, 0.8); + box-shadow: 0 4px 16px rgba(40, 167, 69, 0.2); +} + +.search-input.no-matches { + border-color: rgba(220, 53, 69, 0.8); + box-shadow: 0 4px 16px rgba(220, 53, 69, 0.2); +} + +.search-clear { + position: absolute; + right: 10px; + top: 50%; + transform: translateY(-50%); + width: 20px; + height: 20px; + padding: 0; + display: none; + align-items: center; + justify-content: center; + font-size: 14px; + line-height: 1; + color: #6c757d; + background: transparent; + border: none; + border-radius: 50%; + cursor: pointer; + transition: color var(--transition-fast); +} + +.search-clear:hover { + color: #2e3338; +} + +[data-theme="dark"] .search-clear { + color: #8b949e; +} + +[data-theme="dark"] .search-clear:hover { + color: #e6edf3; +} + +.search-wrapper.has-value .search-clear { + display: flex; +} + +/* Toolbar */ +.toolbar { + display: flex; + align-items: center; + gap: 6px; + margin-left: auto; +} + +.toolbar-btn { display: flex; align-items: center; justify-content: center; - gap: 16px; - font-size: 13px; - box-shadow: 0 2px 8px rgba(55, 118, 171, 0.2); + width: 32px; + height: 32px; + padding: 0; + font-size: 15px; + color: white; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); } -.thread-stat-item { - display: inline-flex; +.toolbar-btn:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +.toolbar-btn:active { + transform: scale(0.95); +} + +/* -------------------------------------------------------------------------- + Sidebar + -------------------------------------------------------------------------- */ + +.sidebar { + width: var(--sidebar-width); + background: var(--bg-secondary); + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + flex-shrink: 0; + overflow: hidden; + position: relative; +} + +.sidebar.collapsed { + width: var(--sidebar-collapsed) !important; + transition: width var(--transition-normal); +} + +.sidebar-toggle { + position: absolute; + top: 12px; + right: 10px; + width: 26px; + height: 26px; + display: flex; + align-items: center; + justify-content: center; + color: var(--text-muted); + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); + z-index: 10; +} + +.sidebar-toggle svg { + transition: transform var(--transition-fast); +} + +.sidebar-toggle:hover { + color: var(--accent); + border-color: var(--accent); + background: var(--accent-glow); +} + +.sidebar.collapsed .sidebar-toggle { + right: 9px; +} + +.sidebar.collapsed .sidebar-toggle svg { + transform: rotate(180deg); +} + +.sidebar-content { + flex: 1; + overflow-y: auto; + padding: 44px 14px 14px; +} + +.sidebar.collapsed .sidebar-content { + display: none; +} + +.sidebar-resize-handle { + position: absolute; + top: 0; + right: 0; + width: 6px; + height: 100%; + cursor: col-resize; + background: transparent; + transition: background var(--transition-fast); + z-index: 11; +} + +.sidebar-resize-handle:hover, +.sidebar-resize-handle.resizing { + background: var(--python-gold); +} + +.sidebar-resize-handle::before { + content: ''; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + width: 2px; + height: 40px; + background: var(--border); + border-radius: 1px; + opacity: 0; + transition: opacity var(--transition-fast); +} + +.sidebar-resize-handle:hover::before { + opacity: 1; +} + +.sidebar.collapsed .sidebar-resize-handle { + display: none; +} + +body.resizing-sidebar { + cursor: col-resize; + user-select: none; +} + +/* Sidebar Logo */ +.sidebar-logo { + display: flex; + justify-content: center; + margin-bottom: 16px; +} + +.sidebar-logo-img { + width: 90px; + height: 90px; + display: flex; + align-items: center; + justify-content: center; +} + +.sidebar-logo-img svg, +.sidebar-logo-img img { + width: 100%; + height: 100%; + object-fit: contain; +} + +/* Sidebar sections */ +.sidebar-section { + margin-bottom: 20px; +} + +.sidebar-section:last-child { + margin-bottom: 0; +} + +.section-title { + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.8px; + color: var(--accent); + margin: 0; + flex: 1; +} + +/* Collapsible sections */ +.collapsible .section-header { + display: flex; + align-items: center; + width: 100%; + padding: 0 0 8px 0; + margin-bottom: 10px; + background: none; + border: none; + border-bottom: 2px solid var(--python-gold); + cursor: pointer; + transition: all var(--transition-fast); +} + +.collapsible .section-header:hover { + opacity: 0.8; +} + +.section-chevron { + color: var(--text-muted); + transition: transform var(--transition-fast); +} + +.collapsible.collapsed .section-chevron { + transform: rotate(-90deg); +} + +.section-content { + overflow: hidden; + transition: max-height var(--transition-normal), opacity var(--transition-normal); + max-height: 1000px; + opacity: 1; +} + +.collapsible.collapsed .section-content { + max-height: 0; + opacity: 0; + margin-bottom: -10px; +} + +/* -------------------------------------------------------------------------- + Profile Summary Cards + -------------------------------------------------------------------------- */ + +.summary-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 6px; +} + +.summary-card { + display: flex; align-items: center; gap: 8px; - background: white; - padding: 6px 14px; - border-radius: 4px; - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08); - transition: all 0.3s ease; - border: 2px solid; - min-width: 115px; - justify-content: center; - animation: fadeIn 0.5s ease-out backwards; + padding: 8px 10px; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + transition: all var(--transition-fast); + animation: slideUp 0.4s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.05s); + overflow: hidden; } -.thread-stat-item:nth-child(1) { animation-delay: 0s; } -.thread-stat-item:nth-child(3) { animation-delay: 0.1s; } -.thread-stat-item:nth-child(5) { animation-delay: 0.2s; } -.thread-stat-item:nth-child(7) { animation-delay: 0.3s; } +.summary-card:nth-child(1) { --i: 0; } +.summary-card:nth-child(2) { --i: 1; } +.summary-card:nth-child(3) { --i: 2; } +.summary-card:nth-child(4) { --i: 3; } + +.summary-card:hover { + border-color: var(--accent); + background: var(--accent-glow); +} + +.summary-icon { + font-size: 16px; + width: 28px; + height: 28px; + display: flex; + align-items: center; + justify-content: center; + background: var(--bg-tertiary); + border-radius: 6px; + flex-shrink: 0; +} + +.summary-data { + min-width: 0; + flex: 1; + overflow: hidden; +} + +.summary-value { + font-family: var(--font-mono); + font-size: 13px; + font-weight: 700; + color: var(--accent); + line-height: 1.2; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.summary-label { + font-size: 8px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.2px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +/* Efficiency Bar */ +.efficiency-section { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border); +} + +.efficiency-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 5px; +} + +.efficiency-label { + font-size: 9px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.2px; +} + +.efficiency-value { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 700; + color: var(--accent); +} + +.efficiency-bar { + height: 6px; + background: var(--bg-tertiary); + border-radius: 3px; + overflow: hidden; +} + +.efficiency-fill { + height: 100%; + background: linear-gradient(90deg, #28a745 0%, #20c997 50%, #17a2b8 100%); + border-radius: 3px; + transition: width 0.6s ease-out; + position: relative; + overflow: hidden; +} + +.efficiency-fill::after { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.4) 50%, + transparent 100% + ); + animation: shimmer 2s ease-in-out infinite; +} + +@keyframes shimmer { + 0% { left: -100%; } + 100% { left: 100%; } +} + +/* -------------------------------------------------------------------------- + Thread Stats Grid (in Sidebar) + -------------------------------------------------------------------------- */ + +.thread-stats-section { + display: block; +} + +.stats-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} + +.stat-tile { + background: var(--bg-primary); + border-radius: 8px; + padding: 10px; + text-align: center; + border: 2px solid var(--border); + transition: all var(--transition-fast); + animation: fadeIn 0.4s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.05s); +} + +.stat-tile:nth-child(1) { --i: 0; } +.stat-tile:nth-child(2) { --i: 1; } +.stat-tile:nth-child(3) { --i: 2; } +.stat-tile:nth-child(4) { --i: 3; } + +.stat-tile:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-sm); +} + +.stat-tile-value { + font-family: var(--font-mono); + font-size: 16px; + font-weight: 700; + color: var(--text-primary); + line-height: 1.2; +} + +.stat-tile-label { + font-size: 9px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.3px; + color: var(--text-muted); + margin-top: 2px; +} + +/* Stat tile color variants */ +.stat-tile--green { --tile-color: 40, 167, 69; --tile-text: #28a745; } +.stat-tile--red { --tile-color: 220, 53, 69; --tile-text: #dc3545; } +.stat-tile--yellow { --tile-color: 255, 193, 7; --tile-text: #d39e00; } +.stat-tile--purple { --tile-color: 111, 66, 193; --tile-text: #6f42c1; } + +.stat-tile[class*="--"] { + border-color: rgba(var(--tile-color), 0.4); + background: linear-gradient(135deg, rgba(var(--tile-color), 0.08) 0%, var(--bg-primary) 100%); +} +.stat-tile[class*="--"] .stat-tile-value { color: var(--tile-text); } + +/* -------------------------------------------------------------------------- + Hotspot Cards + -------------------------------------------------------------------------- */ + +.hotspot { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 10px; + margin-bottom: 8px; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + cursor: pointer; + transition: all var(--transition-fast); + opacity: 0; + transform: translateY(8px); + box-shadow: var(--shadow-sm); +} + +.hotspot.visible { + opacity: 1; + transform: translateY(0); +} + +.hotspot:hover { + border-color: var(--accent); + box-shadow: var(--shadow-md); + transform: translateY(-2px); +} + +.hotspot.active { + border-color: var(--python-gold); + background: var(--accent-glow); + box-shadow: 0 0 0 3px var(--accent-glow); +} + +.hotspot:last-child { + margin-bottom: 0; +} + +.hotspot-rank { + width: 26px; + height: 26px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + font-size: 12px; + flex-shrink: 0; + background: linear-gradient(135deg, var(--python-blue) 0%, var(--python-blue-light) 100%); + color: white; + box-shadow: 0 2px 4px rgba(55, 118, 171, 0.3); +} + +.hotspot-rank--1 { background: linear-gradient(135deg, #d4af37, #f4d03f); color: #5a4a00; } +.hotspot-rank--2 { background: linear-gradient(135deg, #a8a8a8, #c0c0c0); color: #4a4a4a; } +.hotspot-rank--3 { background: linear-gradient(135deg, #cd7f32, #e6a55a); color: #5a3d00; } + +.hotspot-info { + flex: 1; + min-width: 0; +} + +.hotspot-func { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 600; + color: var(--text-primary); + line-height: 1.3; + word-break: break-word; + margin-bottom: 2px; +} + +.hotspot-file { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-muted); + margin-bottom: 3px; + word-break: break-all; +} + +.hotspot-stats { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-secondary); +} + +.hotspot-percent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Legend + -------------------------------------------------------------------------- */ + +.legend-section { + margin-top: auto; + padding-top: 12px; +} + +.legend { + display: flex; + flex-direction: column; + gap: 4px; +} + +.legend-item { + display: flex; + align-items: center; + gap: 8px; + padding: 5px 8px; + background: var(--bg-primary); + border-radius: 4px; + border: 1px solid var(--border-subtle); + font-size: 11px; +} + +.legend-color { + width: 20px; + height: 10px; + border-radius: 2px; + flex-shrink: 0; + border: 1px solid rgba(0, 0, 0, 0.08); +} + +.legend-label { + color: var(--text-primary); + font-weight: 500; + flex: 1; +} + +.legend-range { + font-family: var(--font-mono); + font-size: 9px; + color: var(--text-muted); +} + +/* -------------------------------------------------------------------------- + Thread Filter + -------------------------------------------------------------------------- */ + +.filter-section { + padding-top: 12px; + border-top: 1px solid var(--border); +} + +.filter-label { + display: block; + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + margin-bottom: 6px; +} + +.filter-select { + width: 100%; + padding: 7px 28px 7px 10px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-primary); + background: var(--bg-primary); + border: 2px solid var(--accent); + border-radius: 6px; + cursor: pointer; + outline: none; + appearance: none; + background-image: url("data:image/svg+xml;charset=UTF-8,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%233776ab' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3e%3cpolyline points='6 9 12 15 18 9'%3e%3c/polyline%3e%3c/svg%3e"); + background-repeat: no-repeat; + background-position: right 6px center; + background-size: 14px; + transition: all var(--transition-fast); +} + +.filter-select:hover { + border-color: var(--accent-hover); + box-shadow: 0 2px 6px var(--accent-glow); +} + +.filter-select:focus { + border-color: var(--accent); + box-shadow: 0 0 0 3px var(--accent-glow); +} + +/* -------------------------------------------------------------------------- + Chart Area + -------------------------------------------------------------------------- */ + +.chart-area { + flex: 1; + min-width: 0; + overflow: hidden; + background: var(--bg-primary); + position: relative; +} + +#chart { + width: 100%; + height: 100%; + padding: 16px; + overflow: auto; +} + +/* D3 Flamegraph overrides */ +.d3-flame-graph rect { + stroke: rgba(55, 118, 171, 0.3); + stroke-width: 1px; + cursor: pointer; + transition: filter 0.1s ease; +} + +.d3-flame-graph rect:hover { + stroke: var(--python-blue); + stroke-width: 2px; + filter: brightness(1.08); +} + +.d3-flame-graph text { + font-family: var(--font-sans); + font-size: 12px; + font-weight: 500; + fill: var(--text-primary); + pointer-events: none; +} + +/* Search highlight */ +.d3-flame-graph rect.search-match { + stroke: #ff6b35 !important; + stroke-width: 2px !important; + stroke-dasharray: 4 2; +} + +.d3-flame-graph rect.search-dim { + opacity: 0.25; +} + +/* -------------------------------------------------------------------------- + Status Bar + -------------------------------------------------------------------------- */ + +.status-bar { + height: var(--statusbar-height); + background: var(--bg-secondary); + border-top: 1px solid var(--border); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-secondary); + flex-shrink: 0; +} + +.status-item { + display: flex; + align-items: center; + gap: 5px; +} + +.status-item::before { + content: ''; + width: 4px; + height: 4px; + background: var(--python-gold); + border-radius: 50%; +} + +.status-item:first-child::before { + display: none; +} + +.status-label { + color: var(--text-muted); +} + +.status-value { + color: var(--text-primary); + font-weight: 500; +} + +.status-value.accent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Tooltip + -------------------------------------------------------------------------- */ + +.python-tooltip { + position: absolute; + z-index: 1000; + pointer-events: none; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + padding: 14px; + max-width: 480px; + box-shadow: var(--shadow-lg); + font-family: var(--font-sans); + font-size: 13px; + color: var(--text-primary); + word-wrap: break-word; + overflow-wrap: break-word; + line-height: 1.5; +} + +.tooltip-header { + margin-bottom: 10px; +} + +.tooltip-title { + font-size: 14px; + font-weight: 600; + color: var(--accent); + line-height: 1.3; + word-break: break-word; + margin-bottom: 4px; +} + +.tooltip-location { + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-muted); + background: var(--bg-tertiary); + padding: 4px 8px; + border-radius: 4px; + word-break: break-all; +} + +.tooltip-stats { + display: grid; + grid-template-columns: auto 1fr; + gap: 4px 14px; + font-size: 12px; +} + +.tooltip-stat-label { + color: var(--text-secondary); + font-weight: 500; +} + +.tooltip-stat-value { + color: var(--text-primary); + font-weight: 600; +} + +.tooltip-stat-value.accent { + color: var(--accent); +} + +.tooltip-source { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border); +} + +.tooltip-source-title { + font-size: 11px; + font-weight: 600; + color: var(--accent); + margin-bottom: 6px; +} + +.tooltip-source-code { + font-family: var(--font-mono); + font-size: 10px; + line-height: 1.5; + background: var(--bg-tertiary); + border: 1px solid var(--border); + border-radius: 6px; + padding: 8px; + max-height: 140px; + overflow-y: auto; + white-space: pre-wrap; + word-break: break-all; +} + +.tooltip-source-line { + color: var(--text-secondary); + padding: 1px 0; +} + +.tooltip-source-line.current { + color: var(--accent); + font-weight: 600; +} + +.tooltip-hint { + margin-top: 10px; + padding-top: 8px; + border-top: 1px solid var(--border); + font-size: 11px; + color: var(--text-muted); + text-align: center; +} + +/* -------------------------------------------------------------------------- + Animations + -------------------------------------------------------------------------- */ @keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } + from { opacity: 0; } + to { opacity: 1; } } @keyframes slideUp { from { opacity: 0; - transform: translateY(15px); + transform: translateY(12px); } to { opacity: 1; @@ -160,502 +1156,58 @@ @keyframes slideUp { } } -@keyframes gentlePulse { - 0%, 100% { box-shadow: 0 2px 8px rgba(55, 118, 171, 0.15); } - 50% { box-shadow: 0 2px 16px rgba(55, 118, 171, 0.4); } -} +/* -------------------------------------------------------------------------- + Focus States (Accessibility) + -------------------------------------------------------------------------- */ -/* Color-coded borders and subtle glow on hover */ -#gil-held-stat { - --stat-color: 40, 167, 69; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); -} - -#gil-released-stat { - --stat-color: 220, 53, 69; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); -} - -#gil-waiting-stat { - --stat-color: 255, 193, 7; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); -} - -#gc-stat { - --stat-color: 111, 66, 193; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); -} - -#gil-held-stat:hover, -#gil-released-stat:hover, -#gil-waiting-stat:hover, -#gc-stat:hover { - box-shadow: 0 0 12px rgba(var(--stat-color), 0.4), 0 1px 3px rgba(0, 0, 0, 0.08); -} - -.thread-stat-item .stat-label { - color: #5a6c7d; - font-weight: 600; - font-size: 11px; - letter-spacing: 0.3px; -} - -.thread-stat-item .stat-value { - color: #2e3338; - font-weight: 800; - font-size: 14px; - font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; -} - -.thread-stat-separator { - color: rgba(0, 0, 0, 0.15); - font-weight: 300; - font-size: 16px; - position: relative; - z-index: 1; -} - -/* Responsive - stack on small screens */ -@media (max-width: 768px) { - .thread-stats-bar { - flex-wrap: wrap; - gap: 8px; - font-size: 11px; - padding: 10px 16px; - } - - .thread-stat-item { - padding: 4px 10px; - } - - .thread-stat-item .stat-label { - font-size: 11px; - } - - .thread-stat-item .stat-value { - font-size: 12px; - } - - .thread-stat-separator { - display: none; - } -} - -.stat-card { - background: #ffffff; - border: 1px solid #e9ecef; - border-radius: 12px; - padding: 20px; - display: flex; - align-items: flex-start; - gap: 16px; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.06); - transition: all 0.2s ease; - min-height: 120px; - animation: slideUp 0.4s ease-out backwards; -} - -.stat-card:nth-child(1) { animation-delay: 0.1s; } -.stat-card:nth-child(2) { animation-delay: 0.2s; } -.stat-card:nth-child(3) { animation-delay: 0.3s; } - -.stat-card:hover { - box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1); - transform: translateY(-2px); -} - -.stat-icon { - font-size: 32px; - width: 56px; - height: 56px; - display: flex; - align-items: center; - justify-content: center; - background: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); - border-radius: 50%; - flex-shrink: 0; - box-shadow: 0 2px 8px rgba(55, 118, 171, 0.3); -} - -.stat-content { - flex: 1; -} - -.stat-label { - font-size: 14px; - color: #5a6c7d; - font-weight: 500; - margin-bottom: 4px; - text-transform: uppercase; - letter-spacing: 0.5px; -} - -.stat-value { - font-size: 16px; - font-weight: 700; - color: #2e3338; - line-height: 1.3; - margin-bottom: 4px; - word-break: break-word; - overflow-wrap: break-word; -} - -.stat-file { - font-size: 12px; - color: #8b949e; - font-weight: 400; - margin-bottom: 2px; - font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; - word-break: break-word; - overflow-wrap: break-word; -} - -.stat-detail { - font-size: 12px; - color: #5a6c7d; - font-weight: 400; - line-height: 1.4; - font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; - word-break: break-word; - overflow-wrap: break-word; -} - -.controls { - background: #f8f9fa; - border-bottom: 1px solid #e9ecef; - padding: 20px 0; - text-align: center; -} - -.controls-content { - max-width: 1200px; - margin: 0 auto; - padding: 0 24px; - text-align: center; -} - - -.controls button { - background: #3776ab; - color: white; - border: none; - padding: 12px 24px; - margin: 0 8px; - border-radius: 6px; - cursor: pointer; - font-weight: 600; - font-size: 14px; - font-family: inherit; - transition: all 0.2s ease; - box-shadow: 0 2px 4px rgba(55, 118, 171, 0.2); -} - -.controls button:hover { - background: #2d5aa0; - transform: translateY(-1px); - box-shadow: 0 4px 8px rgba(55, 118, 171, 0.3); -} - -.controls button:active { - transform: translateY(1px); - box-shadow: 0 1px 2px rgba(55, 118, 171, 0.2); -} - -.controls button.secondary { - background: #ffd43b; - color: #2e3338; -} - -.controls button.secondary:hover { - background: #ffcd02; -} - -.controls button.secondary:active { - background: #e6b800; -} - -.thread-filter-wrapper { - display: none; - align-items: center; - margin-left: 16px; - background: white; - border-radius: 6px; - padding: 4px 8px 4px 12px; - border: 2px solid #3776ab; - transition: all 0.2s ease; -} - -.thread-filter-wrapper:hover { - border-color: #2d5aa0; - box-shadow: 0 2px 6px rgba(55, 118, 171, 0.2); -} - -.thread-filter-label { - color: #3776ab; - font-size: 14px; - font-weight: 600; - margin-right: 8px; - display: flex; - align-items: center; -} - -.thread-filter-select { - background: transparent; - color: #2e3338; - border: none; - padding: 4px 24px 4px 4px; - font-size: 14px; - font-weight: 600; - cursor: pointer; - min-width: 120px; - font-family: inherit; - appearance: none; - -webkit-appearance: none; - -moz-appearance: none; - background-image: url("data:image/svg+xml;charset=UTF-8,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%233776ab' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3e%3cpolyline points='6 9 12 15 18 9'%3e%3c/polyline%3e%3c/svg%3e"); - background-repeat: no-repeat; - background-position: right 4px center; - background-size: 16px; -} - -.thread-filter-select:focus { - outline: none; -} - -.thread-filter-select:hover { - color: #3776ab; -} - -.thread-filter-select option { - padding: 8px; - background: white; - color: #2e3338; - font-weight: normal; -} - -#chart { - width: 100%; - height: calc(100vh - 160px); - overflow: hidden; - background: #ffffff; - padding: 0 40px; -} - -.d3-flame-graph rect { - /* Prefer selector specificity instead of !important */ - stroke: rgba(55, 118, 171, 0.3); - stroke-width: 1px; - cursor: pointer; - transition: all 0.1s ease; -} - -.d3-flame-graph rect:hover { - stroke: #3776ab; - stroke-width: 2px; - filter: brightness(1.05); -} - -.d3-flame-graph text { - /* Ensure labels use our font without !important */ - font-family: "Source Sans Pro", sans-serif; - font-size: 12px; - font-weight: 500; - fill: #2e3338; - pointer-events: none; -} - -.info-panel { - position: fixed; - bottom: 24px; - left: 84px; /* Leave space for the button */ - background: white; - padding: 24px; - border-radius: 8px; - border: 1px solid #e9ecef; - font-size: 14px; - max-width: 280px; - box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1); - z-index: 1000; - display: none; -} - -.info-panel h3 { - margin: 0 0 16px 0; - color: #3776ab; - font-weight: 600; - font-size: 16px; - border-bottom: 2px solid #ffd43b; - padding-bottom: 8px; -} - -.info-panel p { - margin: 12px 0; - color: #5a6c7d; - line-height: 1.5; -} - -.info-panel strong { - color: #3776ab; -} - -#show-info-btn { - position: fixed; - bottom: 32px; - left: 32px; - z-index: 1100; - width: 44px; - height: 44px; - border-radius: 50%; - background: #3776ab; - color: white; - border: none; - font-size: 24px; - box-shadow: 0 2px 8px rgba(55, 118, 171, 0.15); - cursor: pointer; - display: flex; - align-items: center; - justify-content: center; - transition: background 0.2s, transform 0.2s; - animation: gentlePulse 3s ease-in-out infinite; -} - -#show-info-btn:hover { - background: #2d5aa0; - animation: none; - transform: scale(1.05); -} - -#close-info-btn { - position: absolute; - top: 8px; - right: 12px; - background: none; - border: none; - font-size: 20px; - cursor: pointer; - color: #3776ab; -} - -@media (max-width: 600px) { - .python-logo { height: 48px; } - .python-logo img { height: 48px; } - #show-info-btn { - left: 8px; - bottom: 8px; - } - .info-panel { - left: 60px; /* Still leave space for button */ - bottom: 8px; - max-width: 90vw; - } -} - -.legend-panel { - position: fixed; - top: 24px; - left: 24px; - background: white; - padding: 24px; - border-radius: 8px; - border: 1px solid #e9ecef; - font-size: 14px; - max-width: 320px; - box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1); - display: none; - z-index: 1001; -} - -.legend-panel h3 { - margin: 0 0 20px 0; - color: #3776ab; - font-weight: 600; - font-size: 18px; - text-align: center; - border-bottom: 2px solid #ffd43b; - padding-bottom: 8px; -} - -.legend-item { - display: flex; - align-items: center; - margin: 12px 0; - padding: 10px; - border-radius: 6px; - background: #f8f9fa; - border: 1px solid #e9ecef; -} - -.legend-color { - width: 28px; - height: 18px; - border-radius: 4px; - margin-right: 16px; - border: 1px solid rgba(0, 0, 0, 0.1); - flex-shrink: 0; -} - -.legend-label { - color: #2e3338; - font-weight: 600; - flex: 1; -} - -.legend-description { - color: #5a6c7d; - font-size: 12px; - margin-top: 2px; - font-weight: 400; -} - -.chart-container { - background: #ffffff; - margin: 0; - padding: 12px 0; -} - -/* Tooltip overflow fixes */ -.python-tooltip { - max-width: 500px !important; - word-wrap: break-word !important; - overflow-wrap: break-word !important; - box-sizing: border-box !important; -} - -/* Responsive tooltip adjustments */ -@media (max-width: 768px) { - .python-tooltip { - max-width: calc(100vw - 40px) !important; - max-height: calc(100vh - 80px) !important; - overflow-y: auto !important; - } -} - -@media (max-width: 480px) { - .python-tooltip { - max-width: calc(100vw - 20px) !important; - font-size: 12px !important; - } -} - -/* Accessibility: visible focus states */ button:focus-visible, select:focus-visible, input:focus-visible { - outline: 2px solid #ffd43b; + outline: 2px solid var(--python-gold); outline-offset: 2px; } -/* Smooth panel transitions */ -.legend-panel, -.info-panel { - transition: opacity 0.2s ease, transform 0.2s ease; +/* -------------------------------------------------------------------------- + Responsive + -------------------------------------------------------------------------- */ + +@media (max-width: 900px) { + .sidebar { + position: fixed; + left: 0; + top: var(--topbar-height); + bottom: var(--statusbar-height); + z-index: 100; + box-shadow: var(--shadow-lg); + } + + .sidebar.collapsed { + width: var(--sidebar-collapsed); + } + + .brand-subtitle { + display: none; + } + + .search-wrapper { + max-width: 220px; + } } -.legend-panel[style*="block"], -.info-panel[style*="block"] { - animation: slideUp 0.2s ease-out; +@media (max-width: 600px) { + .toolbar-btn:not(.theme-toggle) { + display: none; + } + + .search-wrapper { + max-width: 160px; + } + + .brand-info { + display: none; + } + + .stats-grid { + grid-template-columns: 1fr; + } } diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 7faac0effbc..7a2b2ef2e31 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -5,93 +5,219 @@ let stringTable = []; let originalData = null; let currentThreadFilter = 'all'; -// Function to resolve string indices to actual strings +// Heat colors are now defined in CSS variables (--heat-1 through --heat-8) +// and automatically switch with theme changes - no JS color arrays needed! + +// ============================================================================ +// String Resolution +// ============================================================================ + function resolveString(index) { - if (typeof index === 'number' && index >= 0 && index < stringTable.length) { - return stringTable[index]; - } - // Fallback for non-indexed strings or invalid indices - return String(index); + if (index === null || index === undefined) { + return null; + } + if (typeof index === 'number' && index >= 0 && index < stringTable.length) { + return stringTable[index]; + } + return String(index); } -// Function to recursively resolve all string indices in flamegraph data function resolveStringIndices(node) { - if (!node) return node; + if (!node) return node; - // Create a copy to avoid mutating the original - const resolved = { ...node }; + const resolved = { ...node }; - // Resolve string fields - if (typeof resolved.name === 'number') { - resolved.name = resolveString(resolved.name); - } - if (typeof resolved.filename === 'number') { - resolved.filename = resolveString(resolved.filename); - } - if (typeof resolved.funcname === 'number') { - resolved.funcname = resolveString(resolved.funcname); - } + if (typeof resolved.name === 'number') { + resolved.name = resolveString(resolved.name); + } + if (typeof resolved.filename === 'number') { + resolved.filename = resolveString(resolved.filename); + } + if (typeof resolved.funcname === 'number') { + resolved.funcname = resolveString(resolved.funcname); + } - // Resolve source lines if present - if (Array.isArray(resolved.source)) { - resolved.source = resolved.source.map(index => - typeof index === 'number' ? resolveString(index) : index - ); - } + if (Array.isArray(resolved.source)) { + resolved.source = resolved.source.map(index => + typeof index === 'number' ? resolveString(index) : index + ); + } - // Recursively resolve children - if (Array.isArray(resolved.children)) { - resolved.children = resolved.children.map(child => resolveStringIndices(child)); - } + if (Array.isArray(resolved.children)) { + resolved.children = resolved.children.map(child => resolveStringIndices(child)); + } - return resolved; + return resolved; } -// Python color palette - cold to hot -const pythonColors = [ - "#fff4bf", // Coldest - light yellow (<1%) - "#ffec9e", // Cold - yellow (1-3%) - "#ffe47d", // Cool - golden yellow (3-6%) - "#ffdc5c", // Medium - golden (6-12%) - "#ffd43b", // Warm - Python gold (12-18%) - "#5592cc", // Hot - light blue (18-35%) - "#4584bb", // Very hot - medium blue (35-60%) - "#3776ab", // Hottest - Python blue (≥60%) -]; +// ============================================================================ +// Theme & UI Controls +// ============================================================================ -function ensureLibraryLoaded() { - if (typeof flamegraph === "undefined") { - console.error("d3-flame-graph library not loaded"); - document.getElementById("chart").innerHTML = - '

Error: d3-flame-graph library failed to load

'; - throw new Error("d3-flame-graph library failed to load"); +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('flamegraph-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } + + // Re-render flamegraph with new theme colors + if (window.flamegraphData && originalData) { + const tooltip = createPythonTooltip(originalData); + const chart = createFlamegraph(tooltip, originalData.value); + renderFlamegraph(chart, window.flamegraphData); } } +function toggleSidebar() { + const sidebar = document.getElementById('sidebar'); + if (sidebar) { + const isCollapsing = !sidebar.classList.contains('collapsed'); + + if (isCollapsing) { + // Save current width before collapsing + const currentWidth = sidebar.offsetWidth; + sidebar.dataset.expandedWidth = currentWidth; + localStorage.setItem('flamegraph-sidebar-width', currentWidth); + } else { + // Restore width when expanding + const savedWidth = sidebar.dataset.expandedWidth || localStorage.getItem('flamegraph-sidebar-width'); + if (savedWidth) { + sidebar.style.width = savedWidth + 'px'; + } + } + + sidebar.classList.toggle('collapsed'); + localStorage.setItem('flamegraph-sidebar', sidebar.classList.contains('collapsed') ? 'collapsed' : 'expanded'); + + // Resize chart after sidebar animation + setTimeout(() => { + resizeChart(); + }, 300); + } +} + +function resizeChart() { + if (window.flamegraphChart && window.flamegraphData) { + const chartArea = document.querySelector('.chart-area'); + if (chartArea) { + window.flamegraphChart.width(chartArea.clientWidth - 32); + d3.select("#chart").datum(window.flamegraphData).call(window.flamegraphChart); + } + } +} + +function toggleSection(sectionId) { + const section = document.getElementById(sectionId); + if (section) { + section.classList.toggle('collapsed'); + // Save state + const collapsedSections = JSON.parse(localStorage.getItem('flamegraph-collapsed-sections') || '{}'); + collapsedSections[sectionId] = section.classList.contains('collapsed'); + localStorage.setItem('flamegraph-collapsed-sections', JSON.stringify(collapsedSections)); + } +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('flamegraph-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } + + // Restore sidebar state + const savedSidebar = localStorage.getItem('flamegraph-sidebar'); + if (savedSidebar === 'collapsed') { + const sidebar = document.getElementById('sidebar'); + if (sidebar) sidebar.classList.add('collapsed'); + } + + // Restore sidebar width + const savedWidth = localStorage.getItem('flamegraph-sidebar-width'); + if (savedWidth) { + const sidebar = document.getElementById('sidebar'); + if (sidebar) { + sidebar.style.width = savedWidth + 'px'; + } + } + + // Restore collapsed sections + const collapsedSections = JSON.parse(localStorage.getItem('flamegraph-collapsed-sections') || '{}'); + for (const [sectionId, isCollapsed] of Object.entries(collapsedSections)) { + if (isCollapsed) { + const section = document.getElementById(sectionId); + if (section) section.classList.add('collapsed'); + } + } +} + +// ============================================================================ +// Status Bar +// ============================================================================ + +function updateStatusBar(nodeData, rootValue) { + const funcname = resolveString(nodeData.funcname) || resolveString(nodeData.name) || "--"; + const filename = resolveString(nodeData.filename) || ""; + const lineno = nodeData.lineno; + const timeMs = (nodeData.value / 1000).toFixed(2); + const percent = rootValue > 0 ? ((nodeData.value / rootValue) * 100).toFixed(1) : "0.0"; + + const locationEl = document.getElementById('status-location'); + const funcItem = document.getElementById('status-func-item'); + const timeItem = document.getElementById('status-time-item'); + const percentItem = document.getElementById('status-percent-item'); + + if (locationEl) locationEl.style.display = filename && filename !== "~" ? 'flex' : 'none'; + if (funcItem) funcItem.style.display = 'flex'; + if (timeItem) timeItem.style.display = 'flex'; + if (percentItem) percentItem.style.display = 'flex'; + + const fileEl = document.getElementById('status-file'); + if (fileEl && filename && filename !== "~") { + const basename = filename.split('/').pop(); + fileEl.textContent = lineno ? `${basename}:${lineno}` : basename; + } + + const funcEl = document.getElementById('status-func'); + if (funcEl) funcEl.textContent = funcname.length > 40 ? funcname.substring(0, 37) + '...' : funcname; + + const timeEl = document.getElementById('status-time'); + if (timeEl) timeEl.textContent = `${timeMs} ms`; + + const percentEl = document.getElementById('status-percent'); + if (percentEl) percentEl.textContent = `${percent}%`; +} + +function clearStatusBar() { + const ids = ['status-location', 'status-func-item', 'status-time-item', 'status-percent-item']; + ids.forEach(id => { + const el = document.getElementById(id); + if (el) el.style.display = 'none'; + }); +} + +// ============================================================================ +// Tooltip +// ============================================================================ + function createPythonTooltip(data) { const pythonTooltip = flamegraph.tooltip.defaultFlamegraphTooltip(); + pythonTooltip.show = function (d, element) { if (!this._tooltip) { - this._tooltip = d3 - .select("body") + this._tooltip = d3.select("body") .append("div") .attr("class", "python-tooltip") - .style("position", "absolute") - .style("padding", "20px") - .style("background", "white") - .style("color", "#2e3338") - .style("border-radius", "8px") - .style("font-size", "14px") - .style("border", "1px solid #e9ecef") - .style("box-shadow", "0 8px 30px rgba(0, 0, 0, 0.15)") - .style("z-index", "1000") - .style("pointer-events", "none") - .style("font-weight", "400") - .style("line-height", "1.5") - .style("max-width", "500px") - .style("word-wrap", "break-word") - .style("overflow-wrap", "break-word") - .style("font-family", "'Source Sans Pro', sans-serif") .style("opacity", 0); } @@ -101,163 +227,153 @@ function createPythonTooltip(data) { const childCount = d.children ? d.children.length : 0; const source = d.data.source; - // Create source code section if available + const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); + const filename = resolveString(d.data.filename) || ""; + const isSpecialFrame = filename === "~"; + + // Build source section let sourceSection = ""; if (source && Array.isArray(source) && source.length > 0) { const sourceLines = source - .map( - (line) => - `
${line - .replace(/&/g, "&") - .replace(//g, ">")}
`, - ) + .map((line) => { + const isCurrent = line.startsWith("→"); + const escaped = line.replace(/&/g, "&").replace(//g, ">"); + return `
${escaped}
`; + }) .join(""); sourceSection = ` -
-
- Source Code: -
-
- ${sourceLines} -
-
`; - } else if (source) { - // Show debug info if source exists but isn't an array - sourceSection = ` -
-
- [Debug] - Source data type: ${typeof source} -
-
- ${JSON.stringify(source, null, 2)} -
+
+
Source Code:
+
${sourceLines}
`; } - // Resolve strings for display - const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); - const filename = resolveString(d.data.filename) || ""; - - // Don't show file location for special frames like and - const isSpecialFrame = filename === "~"; const fileLocationHTML = isSpecialFrame ? "" : ` -
- ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} -
`; +
${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
`; const tooltipHTML = ` -
-
- ${funcname} -
+
+
${funcname}
${fileLocationHTML} -
- Execution Time: - ${timeMs} ms +
+
+ Execution Time: + ${timeMs} ms - Percentage: - ${percentage}% + Percentage: + ${percentage}% - ${calls > 0 ? ` - Function Calls: - ${calls.toLocaleString()} - ` : ''} + ${calls > 0 ? ` + Function Calls: + ${calls.toLocaleString()} + ` : ''} - ${childCount > 0 ? ` - Child Functions: - ${childCount} - ` : ''} -
- ${sourceSection} -
- ${childCount > 0 ? - "Click to focus on this function" : - "Leaf function - no children"} -
+ ${childCount > 0 ? ` + Child Functions: + ${childCount} + ` : ''} +
+ ${sourceSection} +
+ ${childCount > 0 ? "Click to zoom into this function" : "Leaf function - no children"}
`; - // Get mouse position + // Position tooltip const event = d3.event || window.event; const mouseX = event.pageX || event.clientX; const mouseY = event.pageY || event.clientY; + const padding = 12; - // Calculate tooltip dimensions (default to 320px width if not rendered yet) - let tooltipWidth = 320; - let tooltipHeight = 200; - if (this._tooltip && this._tooltip.node()) { - const node = this._tooltip - .style("opacity", 0) - .style("display", "block") - .node(); - tooltipWidth = node.offsetWidth || 320; - tooltipHeight = node.offsetHeight || 200; - this._tooltip.style("display", null); - } + this._tooltip.html(tooltipHTML); - // Calculate horizontal position: if overflow, show to the left of cursor - const padding = 10; - const rightEdge = mouseX + padding + tooltipWidth; - const viewportWidth = window.innerWidth; - let left; - if (rightEdge > viewportWidth) { + // Measure tooltip + const node = this._tooltip.style("display", "block").style("opacity", 0).node(); + const tooltipWidth = node.offsetWidth || 320; + const tooltipHeight = node.offsetHeight || 200; + + // Calculate position + let left = mouseX + padding; + let top = mouseY + padding; + + if (left + tooltipWidth > window.innerWidth) { left = mouseX - tooltipWidth - padding; - if (left < 0) left = padding; // prevent off left edge - } else { - left = mouseX + padding; + if (left < 0) left = padding; } - // Calculate vertical position: if overflow, show above cursor - const bottomEdge = mouseY + padding + tooltipHeight; - const viewportHeight = window.innerHeight; - let top; - if (bottomEdge > viewportHeight) { + if (top + tooltipHeight > window.innerHeight) { top = mouseY - tooltipHeight - padding; - if (top < 0) top = padding; // prevent off top edge - } else { - top = mouseY + padding; + if (top < 0) top = padding; } this._tooltip - .html(tooltipHTML) .style("left", left + "px") .style("top", top + "px") .transition() - .duration(200) + .duration(150) .style("opacity", 1); + + // Update status bar + updateStatusBar(d.data, data.value); }; - // Override the hide method pythonTooltip.hide = function () { if (this._tooltip) { - this._tooltip.transition().duration(200).style("opacity", 0); + this._tooltip.transition().duration(150).style("opacity", 0); } + clearStatusBar(); }; + return pythonTooltip; } +// ============================================================================ +// Flamegraph Creation +// ============================================================================ + +function ensureLibraryLoaded() { + if (typeof flamegraph === "undefined") { + console.error("d3-flame-graph library not loaded"); + document.getElementById("chart").innerHTML = + '
Error: d3-flame-graph library failed to load
'; + throw new Error("d3-flame-graph library failed to load"); + } +} + +const HEAT_THRESHOLDS = [ + [0.6, 8], + [0.35, 7], + [0.18, 6], + [0.12, 5], + [0.06, 4], + [0.03, 3], + [0.01, 2], +]; + +function getHeatLevel(percentage) { + for (const [threshold, level] of HEAT_THRESHOLDS) { + if (percentage >= threshold) return level; + } + return 1; +} + +function getHeatColors() { + const style = getComputedStyle(document.documentElement); + const colors = {}; + for (let i = 1; i <= 8; i++) { + colors[i] = style.getPropertyValue(`--heat-${i}`).trim(); + } + return colors; +} + function createFlamegraph(tooltip, rootValue) { + const chartArea = document.querySelector('.chart-area'); + const width = chartArea ? chartArea.clientWidth - 32 : window.innerWidth - 320; + const heatColors = getHeatColors(); + let chart = flamegraph() - .width(window.innerWidth - 80) + .width(width) .cellHeight(20) .transitionDuration(300) .minFrameSize(1) @@ -265,141 +381,206 @@ function createFlamegraph(tooltip, rootValue) { .inverted(true) .setColorMapper(function (d) { const percentage = d.data.value / rootValue; - let colorIndex; - if (percentage >= 0.6) colorIndex = 7; - else if (percentage >= 0.35) colorIndex = 6; - else if (percentage >= 0.18) colorIndex = 5; - else if (percentage >= 0.12) colorIndex = 4; - else if (percentage >= 0.06) colorIndex = 3; - else if (percentage >= 0.03) colorIndex = 2; - else if (percentage >= 0.01) colorIndex = 1; - else colorIndex = 0; // <1% - return pythonColors[colorIndex]; + const level = getHeatLevel(percentage); + return heatColors[level]; }); + return chart; } function renderFlamegraph(chart, data) { d3.select("#chart").datum(data).call(chart); - window.flamegraphChart = chart; // for controls - window.flamegraphData = data; // for resize/search + window.flamegraphChart = chart; + window.flamegraphData = data; populateStats(data); } -function attachPanelControls() { - const infoBtn = document.getElementById("show-info-btn"); - const infoPanel = document.getElementById("info-panel"); - const closeBtn = document.getElementById("close-info-btn"); - if (infoBtn && infoPanel) { - infoBtn.addEventListener("click", function () { - const isOpen = infoPanel.style.display === "block"; - infoPanel.style.display = isOpen ? "none" : "block"; - }); - } - if (closeBtn && infoPanel) { - closeBtn.addEventListener("click", function () { - infoPanel.style.display = "none"; - }); - } -} +// ============================================================================ +// Search +// ============================================================================ function updateSearchHighlight(searchTerm, searchInput) { d3.selectAll("#chart rect") - .style("stroke", null) - .style("stroke-width", null) - .style("opacity", null); + .classed("search-match", false) + .classed("search-dim", false); + + // Clear active state from all hotspots + document.querySelectorAll('.hotspot').forEach(h => h.classList.remove('active')); + if (searchTerm && searchTerm.length > 0) { - d3.selectAll("#chart rect").style("opacity", 0.3); let matchCount = 0; + d3.selectAll("#chart rect").each(function (d) { if (d && d.data) { const name = resolveString(d.data.name) || ""; const funcname = resolveString(d.data.funcname) || ""; const filename = resolveString(d.data.filename) || ""; + const lineno = d.data.lineno; const term = searchTerm.toLowerCase(); - const matches = - name.toLowerCase().includes(term) || - funcname.toLowerCase().includes(term) || - filename.toLowerCase().includes(term); + + // Check if search term looks like file:line pattern + const fileLineMatch = term.match(/^(.+):(\d+)$/); + let matches = false; + + if (fileLineMatch) { + // Exact file:line matching + const searchFile = fileLineMatch[1]; + const searchLine = parseInt(fileLineMatch[2], 10); + const basename = filename.split('/').pop().toLowerCase(); + matches = basename.includes(searchFile) && lineno === searchLine; + } else { + // Regular substring search + matches = + name.toLowerCase().includes(term) || + funcname.toLowerCase().includes(term) || + filename.toLowerCase().includes(term); + } + if (matches) { matchCount++; - d3.select(this) - .style("opacity", 1) - .style("stroke", "#ff6b35") - .style("stroke-width", "2px") - .style("stroke-dasharray", "3,3"); + d3.select(this).classed("search-match", true); + } else { + d3.select(this).classed("search-dim", true); } } }); + if (searchInput) { - if (matchCount > 0) { - searchInput.style.borderColor = "rgba(40, 167, 69, 0.8)"; - searchInput.style.boxShadow = "0 6px 20px rgba(40, 167, 69, 0.2)"; - } else { - searchInput.style.borderColor = "rgba(220, 53, 69, 0.8)"; - searchInput.style.boxShadow = "0 6px 20px rgba(220, 53, 69, 0.2)"; - } + searchInput.classList.remove("has-matches", "no-matches"); + searchInput.classList.add(matchCount > 0 ? "has-matches" : "no-matches"); } + + // Mark matching hotspot as active + document.querySelectorAll('.hotspot').forEach(h => { + if (h.dataset.searchterm && h.dataset.searchterm.toLowerCase() === searchTerm.toLowerCase()) { + h.classList.add('active'); + } + }); } else if (searchInput) { - searchInput.style.borderColor = "rgba(255, 255, 255, 0.2)"; - searchInput.style.boxShadow = "0 4px 12px rgba(0, 0, 0, 0.1)"; + searchInput.classList.remove("has-matches", "no-matches"); + } +} + +function searchForHotspot(funcname) { + const searchInput = document.getElementById('search-input'); + const searchWrapper = document.querySelector('.search-wrapper'); + if (searchInput) { + // Toggle: if already searching for this term, clear it + if (searchInput.value.trim() === funcname) { + clearSearch(); + } else { + searchInput.value = funcname; + if (searchWrapper) { + searchWrapper.classList.add('has-value'); + } + performSearch(); + } } } function initSearchHandlers() { const searchInput = document.getElementById("search-input"); + const searchWrapper = document.querySelector(".search-wrapper"); if (!searchInput) return; + let searchTimeout; function performSearch() { const term = searchInput.value.trim(); updateSearchHighlight(term, searchInput); + // Toggle has-value class for clear button visibility + if (searchWrapper) { + searchWrapper.classList.toggle("has-value", term.length > 0); + } } + searchInput.addEventListener("input", function () { clearTimeout(searchTimeout); searchTimeout = setTimeout(performSearch, 150); }); + window.performSearch = performSearch; } -function handleResize(chart, data) { +function clearSearch() { + const searchInput = document.getElementById("search-input"); + const searchWrapper = document.querySelector(".search-wrapper"); + if (searchInput) { + searchInput.value = ""; + searchInput.classList.remove("has-matches", "no-matches"); + if (searchWrapper) { + searchWrapper.classList.remove("has-value"); + } + // Clear highlights + d3.selectAll("#chart rect") + .classed("search-match", false) + .classed("search-dim", false); + // Clear active hotspot + document.querySelectorAll('.hotspot').forEach(h => h.classList.remove('active')); + } +} + +// ============================================================================ +// Resize Handler +// ============================================================================ + +function handleResize() { + let resizeTimeout; window.addEventListener("resize", function () { - if (chart && data) { - const newWidth = window.innerWidth - 80; - chart.width(newWidth); - d3.select("#chart").datum(data).call(chart); + clearTimeout(resizeTimeout); + resizeTimeout = setTimeout(resizeChart, 100); + }); +} + +function initSidebarResize() { + const sidebar = document.getElementById('sidebar'); + const resizeHandle = document.getElementById('sidebar-resize-handle'); + if (!sidebar || !resizeHandle) return; + + let isResizing = false; + let startX = 0; + let startWidth = 0; + const minWidth = 200; + const maxWidth = 600; + + resizeHandle.addEventListener('mousedown', function(e) { + isResizing = true; + startX = e.clientX; + startWidth = sidebar.offsetWidth; + resizeHandle.classList.add('resizing'); + document.body.classList.add('resizing-sidebar'); + e.preventDefault(); + }); + + document.addEventListener('mousemove', function(e) { + if (!isResizing) return; + + const deltaX = e.clientX - startX; + const newWidth = Math.min(Math.max(startWidth + deltaX, minWidth), maxWidth); + sidebar.style.width = newWidth + 'px'; + e.preventDefault(); + }); + + document.addEventListener('mouseup', function() { + if (isResizing) { + isResizing = false; + resizeHandle.classList.remove('resizing'); + document.body.classList.remove('resizing-sidebar'); + + // Save the new width + const width = sidebar.offsetWidth; + localStorage.setItem('flamegraph-sidebar-width', width); + + // Resize chart after sidebar resize + setTimeout(() => { + resizeChart(); + }, 10); } }); } -function initFlamegraph() { - ensureLibraryLoaded(); - - // Extract string table if present and resolve string indices - let processedData = EMBEDDED_DATA; - if (EMBEDDED_DATA.strings) { - stringTable = EMBEDDED_DATA.strings; - processedData = resolveStringIndices(EMBEDDED_DATA); - } - - // Store original data for filtering - originalData = processedData; - - // Initialize thread filter dropdown - initThreadFilter(processedData); - - const tooltip = createPythonTooltip(processedData); - const chart = createFlamegraph(tooltip, processedData.value); - renderFlamegraph(chart, processedData); - attachPanelControls(); - initSearchHandlers(); - handleResize(chart, processedData); -} - -if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", initFlamegraph); -} else { - initFlamegraph(); -} +// ============================================================================ +// Thread Stats +// ============================================================================ // Mode constants (must match constants.py) const PROFILING_MODE_WALL = 0; @@ -408,97 +589,164 @@ const PROFILING_MODE_GIL = 2; const PROFILING_MODE_ALL = 3; function populateThreadStats(data, selectedThreadId = null) { - // Check if thread statistics are available const stats = data?.stats; if (!stats || !stats.thread_stats) { - return; // No thread stats available + return; } const mode = stats.mode !== undefined ? stats.mode : PROFILING_MODE_WALL; let threadStats; - // If a specific thread is selected, use per-thread stats if (selectedThreadId !== null && stats.per_thread_stats && stats.per_thread_stats[selectedThreadId]) { threadStats = stats.per_thread_stats[selectedThreadId]; } else { threadStats = stats.thread_stats; } - // Validate threadStats object - if (!threadStats || typeof threadStats.total !== 'number') { - return; // Invalid thread stats + if (!threadStats || typeof threadStats.total !== 'number' || threadStats.total <= 0) { + return; } - const bar = document.getElementById('thread-stats-bar'); - if (!bar) { - return; // DOM element not found + const section = document.getElementById('thread-stats-bar'); + if (!section) { + return; } - // Show the bar if we have valid thread stats - if (threadStats.total > 0) { - bar.style.display = 'flex'; + section.style.display = 'block'; - // Hide/show GIL stats items in GIL mode - const gilHeldStat = document.getElementById('gil-held-stat'); - const gilReleasedStat = document.getElementById('gil-released-stat'); - const gilWaitingStat = document.getElementById('gil-waiting-stat'); - const separators = bar.querySelectorAll('.thread-stat-separator'); + const gilHeldStat = document.getElementById('gil-held-stat'); + const gilReleasedStat = document.getElementById('gil-released-stat'); + const gilWaitingStat = document.getElementById('gil-waiting-stat'); - if (mode === PROFILING_MODE_GIL) { - // In GIL mode, hide GIL-related stats - if (gilHeldStat) gilHeldStat.style.display = 'none'; - if (gilReleasedStat) gilReleasedStat.style.display = 'none'; - if (gilWaitingStat) gilWaitingStat.style.display = 'none'; - separators.forEach((sep, i) => { - if (i < 3) sep.style.display = 'none'; - }); - } else { - // Show all stats in other modes - if (gilHeldStat) gilHeldStat.style.display = 'inline-flex'; - if (gilReleasedStat) gilReleasedStat.style.display = 'inline-flex'; - if (gilWaitingStat) gilWaitingStat.style.display = 'inline-flex'; - separators.forEach(sep => sep.style.display = 'inline'); + if (mode === PROFILING_MODE_GIL) { + // In GIL mode, hide GIL-related stats + if (gilHeldStat) gilHeldStat.style.display = 'none'; + if (gilReleasedStat) gilReleasedStat.style.display = 'none'; + if (gilWaitingStat) gilWaitingStat.style.display = 'none'; + } else { + // Show all stats + if (gilHeldStat) gilHeldStat.style.display = 'block'; + if (gilReleasedStat) gilReleasedStat.style.display = 'block'; + if (gilWaitingStat) gilWaitingStat.style.display = 'block'; - // GIL Held - const gilHeldPct = threadStats.has_gil_pct || 0; - const gilHeldPctElem = document.getElementById('gil-held-pct'); - if (gilHeldPctElem) gilHeldPctElem.textContent = `${gilHeldPct.toFixed(2)}%`; + const gilHeldPctElem = document.getElementById('gil-held-pct'); + if (gilHeldPctElem) gilHeldPctElem.textContent = `${(threadStats.has_gil_pct || 0).toFixed(1)}%`; - // GIL Released (threads running without GIL) - const gilReleasedPct = threadStats.on_cpu_pct || 0; - const gilReleasedPctElem = document.getElementById('gil-released-pct'); - if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${gilReleasedPct.toFixed(2)}%`; + const gilReleasedPctElem = document.getElementById('gil-released-pct'); + if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${(threadStats.on_cpu_pct || 0).toFixed(1)}%`; - // Waiting for GIL - const gilWaitingPct = threadStats.gil_requested_pct || 0; - const gilWaitingPctElem = document.getElementById('gil-waiting-pct'); - if (gilWaitingPctElem) gilWaitingPctElem.textContent = `${gilWaitingPct.toFixed(2)}%`; - } + const gilWaitingPctElem = document.getElementById('gil-waiting-pct'); + if (gilWaitingPctElem) gilWaitingPctElem.textContent = `${(threadStats.gil_requested_pct || 0).toFixed(1)}%`; + } - // Garbage Collection (always show) - const gcPct = threadStats.gc_pct || 0; - const gcPctElem = document.getElementById('gc-pct'); - if (gcPctElem) gcPctElem.textContent = `${gcPct.toFixed(2)}%`; + const gcPctElem = document.getElementById('gc-pct'); + if (gcPctElem) gcPctElem.textContent = `${(threadStats.gc_pct || 0).toFixed(1)}%`; +} + +// ============================================================================ +// Profile Summary Stats +// ============================================================================ + +function formatNumber(num) { + if (num >= 1000000) return (num / 1000000).toFixed(1) + 'M'; + if (num >= 1000) return (num / 1000).toFixed(1) + 'K'; + return num.toLocaleString(); +} + +function formatDuration(seconds) { + if (seconds >= 3600) { + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + return `${h}h ${m}m`; + } + if (seconds >= 60) { + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m}m ${s}s`; + } + return seconds.toFixed(2) + 's'; +} + +function populateProfileSummary(data) { + const stats = data.stats || {}; + const totalSamples = stats.total_samples || data.value || 0; + const duration = stats.duration_sec || 0; + const sampleRate = stats.sample_rate || (duration > 0 ? totalSamples / duration : 0); + const errorRate = stats.error_rate || 0; + const missedSamples= stats.missed_samples || 0; + + const samplesEl = document.getElementById('stat-total-samples'); + if (samplesEl) samplesEl.textContent = formatNumber(totalSamples); + + const durationEl = document.getElementById('stat-duration'); + if (durationEl) durationEl.textContent = duration > 0 ? formatDuration(duration) : '--'; + + const rateEl = document.getElementById('stat-sample-rate'); + if (rateEl) rateEl.textContent = sampleRate > 0 ? formatNumber(Math.round(sampleRate)) : '--'; + + // Count unique functions + let functionCount = 0; + function countFunctions(node) { + if (!node) return; + functionCount++; + if (node.children) node.children.forEach(countFunctions); + } + countFunctions(data); + + const functionsEl = document.getElementById('stat-functions'); + if (functionsEl) functionsEl.textContent = formatNumber(functionCount); + + // Efficiency bar + if (errorRate !== undefined && errorRate !== null) { + const efficiency = Math.max(0, Math.min(100, (100 - errorRate))); + + const efficiencySection = document.getElementById('efficiency-section'); + if (efficiencySection) efficiencySection.style.display = 'block'; + + const efficiencyValue = document.getElementById('stat-efficiency'); + if (efficiencyValue) efficiencyValue.textContent = efficiency.toFixed(1) + '%'; + + const efficiencyFill = document.getElementById('efficiency-fill'); + if (efficiencyFill) efficiencyFill.style.width = efficiency + '%'; + } + // MissedSamples bar + if (missedSamples !== undefined && missedSamples !== null) { + const sampleEfficiency = Math.max(0, missedSamples); + + const efficiencySection = document.getElementById('efficiency-section'); + if (efficiencySection) efficiencySection.style.display = 'block'; + + const sampleEfficiencyValue = document.getElementById('stat-missed-samples'); + if (sampleEfficiencyValue) sampleEfficiencyValue.textContent = sampleEfficiency.toFixed(1) + '%'; + + const sampleEfficiencyFill = document.getElementById('missed-samples-fill'); + if (sampleEfficiencyFill) sampleEfficiencyFill.style.width = sampleEfficiency + '%'; } } +// ============================================================================ +// Hotspot Stats +// ============================================================================ + function populateStats(data) { const totalSamples = data.value || 0; + // Populate profile summary + populateProfileSummary(data); + // Populate thread statistics if available populateThreadStats(data); - // Collect all functions with their metrics, aggregated by function name const functionMap = new Map(); function collectFunctions(node) { if (!node) return; - let filename = typeof node.filename === 'number' ? resolveString(node.filename) : node.filename; - let funcname = typeof node.funcname === 'number' ? resolveString(node.funcname) : node.funcname; + let filename = resolveString(node.filename); + let funcname = resolveString(node.funcname); if (!filename || !funcname) { - const nameStr = typeof node.name === 'number' ? resolveString(node.name) : node.name; + const nameStr = resolveString(node.name); if (nameStr?.includes('(')) { const match = nameStr.match(/^(.+?)\s*\((.+?):(\d+)\)$/); if (match) { @@ -512,21 +760,18 @@ function populateStats(data) { funcname = funcname || 'unknown'; if (filename !== 'unknown' && funcname !== 'unknown' && node.value > 0) { - // Calculate direct samples (this node's value minus children's values) let childrenValue = 0; if (node.children) { childrenValue = node.children.reduce((sum, child) => sum + child.value, 0); } const directSamples = Math.max(0, node.value - childrenValue); - // Use file:line:funcname as key to ensure uniqueness const funcKey = `${filename}:${node.lineno || '?'}:${funcname}`; if (functionMap.has(funcKey)) { const existing = functionMap.get(funcKey); existing.directSamples += directSamples; existing.directPercent = (existing.directSamples / totalSamples) * 100; - // Keep the most representative file/line (the one with more samples) if (directSamples > existing.maxSingleSamples) { existing.filename = filename; existing.lineno = node.lineno || '?'; @@ -551,96 +796,81 @@ function populateStats(data) { collectFunctions(data); - // Convert map to array and get top 3 hotspots const hotSpots = Array.from(functionMap.values()) - .filter(f => f.directPercent > 0.5) // At least 0.5% to be significant + .filter(f => f.directPercent > 0.5) .sort((a, b) => b.directPercent - a.directPercent) .slice(0, 3); - // Populate the 3 cards + // Populate and animate hotspot cards for (let i = 0; i < 3; i++) { const num = i + 1; + const card = document.getElementById(`hotspot-${num}`); + const funcEl = document.getElementById(`hotspot-func-${num}`); + const fileEl = document.getElementById(`hotspot-file-${num}`); + const percentEl = document.getElementById(`hotspot-percent-${num}`); + const samplesEl = document.getElementById(`hotspot-samples-${num}`); + if (i < hotSpots.length && hotSpots[i]) { - const hotspot = hotSpots[i]; - const filename = hotspot.filename || 'unknown'; - const lineno = hotspot.lineno ?? '?'; - let funcDisplay = hotspot.funcname || 'unknown'; - if (funcDisplay.length > 35) { - funcDisplay = funcDisplay.substring(0, 32) + '...'; - } - - // Don't show file:line for special frames like and + const h = hotSpots[i]; + const filename = h.filename || 'unknown'; + const lineno = h.lineno ?? '?'; const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?'); - let fileDisplay; - if (isSpecialFrame) { - fileDisplay = '--'; + + let funcDisplay = h.funcname || 'unknown'; + if (funcDisplay.length > 28) funcDisplay = funcDisplay.substring(0, 25) + '...'; + + if (funcEl) funcEl.textContent = funcDisplay; + if (fileEl) { + if (isSpecialFrame) { + fileEl.textContent = '--'; + } else { + const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; + fileEl.textContent = `${basename}:${lineno}`; + } + } + if (percentEl) percentEl.textContent = `${h.directPercent.toFixed(1)}%`; + if (samplesEl) samplesEl.textContent = ` (${h.directSamples.toLocaleString()})`; + } else { + if (funcEl) funcEl.textContent = '--'; + if (fileEl) fileEl.textContent = '--'; + if (percentEl) percentEl.textContent = '--'; + if (samplesEl) samplesEl.textContent = ''; + } + + // Add click handler and animate entrance + if (card) { + if (i < hotSpots.length && hotSpots[i]) { + const h = hotSpots[i]; + const basename = h.filename !== 'unknown' ? h.filename.split('/').pop() : ''; + const searchTerm = basename && h.lineno !== '?' ? `${basename}:${h.lineno}` : h.funcname; + card.dataset.searchterm = searchTerm; + card.onclick = () => searchForHotspot(searchTerm); + card.style.cursor = 'pointer'; } else { - const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; - fileDisplay = `${basename}:${lineno}`; + card.onclick = null; + delete card.dataset.searchterm; + card.style.cursor = 'default'; } - document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay; - document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay; - document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`; - } else { - document.getElementById(`hotspot-file-${num}`).textContent = '--'; - document.getElementById(`hotspot-func-${num}`).textContent = '--'; - document.getElementById(`hotspot-detail-${num}`).textContent = '--'; + setTimeout(() => { + card.classList.add('visible'); + }, 100 + i * 80); } } } -// Control functions -function resetZoom() { - if (window.flamegraphChart) { - window.flamegraphChart.resetZoom(); - } -} - -function exportSVG() { - const svgElement = document.querySelector("#chart svg"); - if (svgElement) { - const serializer = new XMLSerializer(); - const svgString = serializer.serializeToString(svgElement); - const blob = new Blob([svgString], { type: "image/svg+xml" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = "python-performance-flamegraph.svg"; - a.click(); - URL.revokeObjectURL(url); - } -} - -function toggleLegend() { - const legendPanel = document.getElementById("legend-panel"); - const isHidden = - legendPanel.style.display === "none" || legendPanel.style.display === ""; - legendPanel.style.display = isHidden ? "block" : "none"; -} - -function clearSearch() { - const searchInput = document.getElementById("search-input"); - if (searchInput) { - searchInput.value = ""; - if (window.flamegraphChart) { - window.flamegraphChart.clear(); - } - } -} +// ============================================================================ +// Thread Filter +// ============================================================================ function initThreadFilter(data) { const threadFilter = document.getElementById('thread-filter'); - const threadWrapper = document.querySelector('.thread-filter-wrapper'); + const threadSection = document.getElementById('thread-section'); - if (!threadFilter || !data.threads) { - return; - } + if (!threadFilter || !data.threads) return; - // Clear existing options except "All Threads" threadFilter.innerHTML = ''; - // Add thread options const threads = data.threads || []; threads.forEach(threadId => { const option = document.createElement('option'); @@ -649,9 +879,8 @@ function initThreadFilter(data) { threadFilter.appendChild(option); }); - // Show filter if more than one thread - if (threads.length > 1 && threadWrapper) { - threadWrapper.style.display = 'inline-flex'; + if (threads.length > 1 && threadSection) { + threadSection.style.display = 'block'; } } @@ -666,11 +895,9 @@ function filterByThread() { let selectedThreadId = null; if (selectedThread === 'all') { - // Show all data filteredData = originalData; } else { - // Filter data by thread - selectedThreadId = parseInt(selectedThread); + selectedThreadId = parseInt(selectedThread, 10); filteredData = filterDataByThread(originalData, selectedThreadId); if (filteredData.strings) { @@ -679,12 +906,10 @@ function filterByThread() { } } - // Re-render flamegraph with filtered data const tooltip = createPythonTooltip(filteredData); const chart = createFlamegraph(tooltip, filteredData.value); renderFlamegraph(chart, filteredData); - // Update thread stats to show per-thread or aggregate stats populateThreadStats(originalData, selectedThreadId); } @@ -694,10 +919,7 @@ function filterDataByThread(data, threadId) { return null; } - const filteredNode = { - ...node, - children: [] - }; + const filteredNode = { ...node, children: [] }; if (node.children && Array.isArray(node.children)) { filteredNode.children = node.children @@ -708,17 +930,6 @@ function filterDataByThread(data, threadId) { return filteredNode; } - const filteredRoot = { - ...data, - children: [] - }; - - if (data.children && Array.isArray(data.children)) { - filteredRoot.children = data.children - .map(child => filterNode(child)) - .filter(child => child !== null); - } - function recalculateValue(node) { if (!node.children || node.children.length === 0) { return node.value || 0; @@ -728,8 +939,72 @@ function filterDataByThread(data, threadId) { return node.value; } - recalculateValue(filteredRoot); + const filteredRoot = { ...data, children: [] }; + if (data.children && Array.isArray(data.children)) { + filteredRoot.children = data.children + .map(child => filterNode(child)) + .filter(child => child !== null); + } + + recalculateValue(filteredRoot); return filteredRoot; } +// ============================================================================ +// Control Functions +// ============================================================================ + +function resetZoom() { + if (window.flamegraphChart) { + window.flamegraphChart.resetZoom(); + } +} + +function exportSVG() { + const svgElement = document.querySelector("#chart svg"); + if (!svgElement) { + console.warn("Cannot export: No flamegraph SVG found"); + return; + } + const serializer = new XMLSerializer(); + const svgString = serializer.serializeToString(svgElement); + const blob = new Blob([svgString], { type: "image/svg+xml" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "python-performance-flamegraph.svg"; + a.click(); + URL.revokeObjectURL(url); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +function initFlamegraph() { + ensureLibraryLoaded(); + restoreUIState(); + + let processedData = EMBEDDED_DATA; + if (EMBEDDED_DATA.strings) { + stringTable = EMBEDDED_DATA.strings; + processedData = resolveStringIndices(EMBEDDED_DATA); + } + + originalData = processedData; + initThreadFilter(processedData); + + const tooltip = createPythonTooltip(processedData); + const chart = createFlamegraph(tooltip, processedData.value); + renderFlamegraph(chart, processedData); + initSearchHandlers(); + initSidebarResize(); + handleResize(); +} + +if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", initFlamegraph); +} else { + initFlamegraph(); +} diff --git a/Lib/profiling/sampling/flamegraph_template.html b/Lib/profiling/sampling/flamegraph_template.html index 5f94bbe69c4..09b673b76da 100644 --- a/Lib/profiling/sampling/flamegraph_template.html +++ b/Lib/profiling/sampling/flamegraph_template.html @@ -1,9 +1,9 @@ - + - Python Performance Flamegraph + Tachyon Profiler - Flamegraph @@ -11,165 +11,297 @@ -
-
- -
-

Tachyon Profiler Performance Flamegraph

-
- Interactive visualization of function call performance +
+ +
+
+ Tachyon + + Profiler +
+
+ + +
+
+ + + +
+
+ + +
+ +
-
- -
-
-
+ - - + +
+
+
+
-
- -
-
-
🥇
-
-
#1 Hot Spot
-
--
-
--
-
--
-
-
-
-
🥈
-
-
#2 Hot Spot
-
--
-
--
-
--
-
-
-
-
🥉
-
-
#3 Hot Spot
-
--
-
--
-
--
-
-
-
-
- -
-
- - - -
- - -
-
-
- - - -
- -

Navigation Guide

-

Click: Zoom into function

-

Hover: Show detailed information

-

Width: Time spent in function

-

Height: Call stack depth

-

Color: Performance intensity

-
- -
-

🔥 Performance Heat Map

-
-
-
-
Hottest Functions (≥60%)
-
Highest performance impact
-
-
-
-
-
-
Very Hot Functions (35-60%)
-
High performance impact
-
-
-
-
-
-
Hot Functions (18-35%)
-
Notable performance cost
-
-
-
-
-
-
Warm Functions (12-18%)
-
Moderate impact
-
-
-
-
-
-
Medium Functions (6-12%)
-
Some performance impact
-
-
-
-
-
-
Cool Functions (3-6%)
-
Low performance impact
-
-
-
-
-
-
Cold Functions (1-3%)
-
Minimal performance impact
-
-
-
-
-
-
Coldest Functions (<1%)
-
Very low performance impact
-
-
-
- -
-
+ +
+ + + + +
diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 82c0d3959ba..88d9a4fa13b 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -112,8 +112,10 @@ def sample(self, collector, duration_sec=10): if self.realtime_stats and len(self.sample_intervals) > 0: print() # Add newline after real-time stats - sample_rate = num_samples / running_time + sample_rate = num_samples / running_time if running_time > 0 else 0 error_rate = (errors / num_samples) * 100 if num_samples > 0 else 0 + expected_samples = int(duration_sec / sample_interval_sec) + missed_samples = (expected_samples - num_samples) / expected_samples * 100 if expected_samples > 0 else 0 # Don't print stats for live mode (curses is handling display) is_live_mode = LiveStatsCollector is not None and isinstance(collector, LiveStatsCollector) @@ -124,9 +126,8 @@ def sample(self, collector, duration_sec=10): # Pass stats to flamegraph collector if it's the right type if hasattr(collector, 'set_stats'): - collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, mode=self.mode) + collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, missed_samples, mode=self.mode) - expected_samples = int(duration_sec / sample_interval_sec) if num_samples < expected_samples and not is_live_mode and not interrupted: print( f"Warning: missed {expected_samples - num_samples} samples " diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 9028a8bebb1..146a058a03a 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -113,13 +113,15 @@ def collect(self, stack_frames, skip_idle=False): # Call parent collect to process frames super().collect(stack_frames, skip_idle=skip_idle) - def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, mode=None): + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, + error_rate=None, missed_samples=None, mode=None): """Set profiling statistics to include in flamegraph data.""" self.stats = { "sample_interval_usec": sample_interval_usec, "duration_sec": duration_sec, "sample_rate": sample_rate, "error_rate": error_rate, + "missed_samples": missed_samples, "mode": mode } diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 38665f5a591..e8c12c22215 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -494,7 +494,7 @@ def test_flamegraph_collector_export(self): # Should be valid HTML self.assertIn("", content.lower()) self.assertIn(" Date: Mon, 1 Dec 2025 18:37:46 +0000 Subject: [PATCH 375/417] gh-138122: Small fixes to the new tachyon UI (#142157) --- Lib/profiling/sampling/flamegraph.js | 4 +++- Lib/profiling/sampling/flamegraph_template.html | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 7a2b2ef2e31..494d156a8dd 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -633,7 +633,9 @@ function populateThreadStats(data, selectedThreadId = null) { if (gilHeldPctElem) gilHeldPctElem.textContent = `${(threadStats.has_gil_pct || 0).toFixed(1)}%`; const gilReleasedPctElem = document.getElementById('gil-released-pct'); - if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${(threadStats.on_cpu_pct || 0).toFixed(1)}%`; + // GIL Released = not holding GIL and not waiting for it + const gilReleasedPct = Math.max(0, 100 - (threadStats.has_gil_pct || 0) - (threadStats.gil_requested_pct || 0)); + if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${gilReleasedPct.toFixed(1)}%`; const gilWaitingPctElem = document.getElementById('gil-waiting-pct'); if (gilWaitingPctElem) gilWaitingPctElem.textContent = `${(threadStats.gil_requested_pct || 0).toFixed(1)}%`; diff --git a/Lib/profiling/sampling/flamegraph_template.html b/Lib/profiling/sampling/flamegraph_template.html index 09b673b76da..82102c229e7 100644 --- a/Lib/profiling/sampling/flamegraph_template.html +++ b/Lib/profiling/sampling/flamegraph_template.html @@ -155,7 +155,7 @@

Runtime Stats

--
-
Waiting
+
Waiting GIL
--
From eb892868b31322d7cf271bc25923e14b1f67ae38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 1 Dec 2025 19:04:47 -0500 Subject: [PATCH 376/417] gh-142048: Fix quadratically increasing GC delays (gh-142051) The GC for the free threaded build would get slower with each collection due to effectively double counting objects freed by the GC. --- .../2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst | 2 ++ Python/gc_free_threading.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst new file mode 100644 index 00000000000..1400dae13ff --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst @@ -0,0 +1,2 @@ +Fix quadratically increasing garbage collection delays in free-threaded +build. diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 1717603b947..e672e870db2 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2210,7 +2210,19 @@ record_deallocation(PyThreadState *tstate) gc->alloc_count--; if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int new_count; + do { + if (count == 0) { + break; + } + new_count = count + (int)gc->alloc_count; + if (new_count < 0) { + new_count = 0; + } + } while (!_Py_atomic_compare_exchange_int(&gcstate->young.count, + &count, + new_count)); gc->alloc_count = 0; } } From 41728856a277749d022d99af82c8e14b85057f3c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 1 Dec 2025 23:13:11 -0500 Subject: [PATCH 377/417] gh-142163: Only define `HAVE_THREAD_LOCAL` when `Py_BUILD_CORE` is set (#142164) --- Include/pyport.h | 10 ++++++++-- .../2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst diff --git a/Include/pyport.h b/Include/pyport.h index 97c0e195d19..61e2317976e 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -509,9 +509,15 @@ extern "C" { # define Py_CAN_START_THREADS 1 #endif -#ifdef WITH_THREAD -// HAVE_THREAD_LOCAL is just defined here for compatibility's sake + +/* gh-142163: Some libraries rely on HAVE_THREAD_LOCAL being undefined, so + * we can only define it only when Py_BUILD_CORE is set.*/ +#ifdef Py_BUILD_CORE +// This is no longer coupled to _Py_thread_local. # define HAVE_THREAD_LOCAL 1 +#endif + +#ifdef WITH_THREAD # ifdef thread_local # define _Py_thread_local thread_local # elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) diff --git a/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst b/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst new file mode 100644 index 00000000000..5edcfd81992 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst @@ -0,0 +1,2 @@ +Fix the ``HAVE_THREAD_LOCAL`` macro being defined without the +``Py_BUILD_CORE`` macro set after including :file:`Python.h`. From 5e58548ebe8f7ac8c6cb0bad775912caa4090515 Mon Sep 17 00:00:00 2001 From: LloydZ <35182391+cocolato@users.noreply.github.com> Date: Tue, 2 Dec 2025 12:41:54 +0800 Subject: [PATCH 378/417] gh-59000: Fix pdb breakpoint resolution for class methods when module not imported (#141949) --- Lib/pdb.py | 4 +++- Lib/test/test_pdb.py | 16 ++++++++++++++++ ...2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 60b713ebaf3..18cee7e9ae6 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1487,7 +1487,9 @@ def lineinfo(self, identifier): f = self.lookupmodule(parts[0]) if f: fname = f - item = parts[1] + item = parts[1] + else: + return failed answer = find_function(item, self.canonic(fname)) return answer or failed diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 34dfc220c7e..8d582742499 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -4587,6 +4587,22 @@ def bar(): ])) self.assertIn('break in bar', stdout) + def test_issue_59000(self): + script = """ + def foo(): + pass + + class C: + def foo(self): + pass + """ + commands = """ + break C.foo + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("The specified object 'C.foo' is not a function", stdout) + class ChecklineTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst b/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst new file mode 100644 index 00000000000..33ab8a0659e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst @@ -0,0 +1 @@ +Fix :mod:`pdb` breakpoint resolution for class methods when the module defining the class is not imported. From 2dc28eb8b0956f3ebc8ec9cedcb614fb7516e120 Mon Sep 17 00:00:00 2001 From: Krishna-web-hub Date: Tue, 2 Dec 2025 11:23:12 +0530 Subject: [PATCH 379/417] gh-140281: Update free threading Python HOWTO for 3.14 (gh-140566) Co-authored-by: Sam Gross --- Doc/howto/free-threading-python.rst | 57 +++++++++-------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index e4df7a787a2..380c2be0495 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -11,9 +11,7 @@ available processing power by running threads in parallel on available CPU cores While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -The free-threaded mode is working and continues to be improved, but -there is some additional overhead in single-threaded workloads compared -to the regular build. Additionally, third-party packages, in particular ones +Some third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. @@ -101,63 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage under Python 3.13. This -has been addressed in the 3.14 release, where the aforementioned objects use -deferred reference counting to avoid reference count contention. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized in the 3.13 release. This -behavior is part of the 3.14 release as well and it is expected to remain in -future free-threaded builds. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. - -The specializing adaptive interpreter has been re-enabled in a thread-safe way -in the 3.14 release. The performance penalty on single-threaded code in -free-threaded mode is now roughly 5-10%, depending on the platform and C -compiler used. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes From fddc24e4c85467f14075e94fd0d23d928ceb535f Mon Sep 17 00:00:00 2001 From: LloydZ <35182391+cocolato@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:40:02 +0800 Subject: [PATCH 380/417] gh-141982: Fix pdb can't set breakpoints on async functions (#141983) Co-authored-by: Tian Gao --- Lib/pdb.py | 2 +- Lib/test/test_pdb.py | 19 +++++++++++++++++++ ...-11-30-04-28-30.gh-issue-141982.pxZct9.rst | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 18cee7e9ae6..1506e3d4709 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -130,7 +130,7 @@ def find_first_executable_line(code): return code.co_firstlineno def find_function(funcname, filename): - cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) + cre = re.compile(r'(?:async\s+)?def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 8d582742499..c097808e7fd 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -4587,6 +4587,25 @@ def bar(): ])) self.assertIn('break in bar', stdout) + @unittest.skipIf(SKIP_CORO_TESTS, "Coroutine tests are skipped") + def test_async_break(self): + script = """ + import asyncio + + async def main(): + pass + + asyncio.run(main()) + """ + commands = """ + break main + continue + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertRegex(stdout, r"Breakpoint 1 at .*main\.py:5") + self.assertIn("pass", stdout) + def test_issue_59000(self): script = """ def foo(): diff --git a/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst b/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst new file mode 100644 index 00000000000..e5ec593dd6e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst @@ -0,0 +1 @@ +Allow :mod:`pdb` to set breakpoints on async functions with function names. From 748c4b47b70f89cb132f865250eb6c951a177366 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 2 Dec 2025 09:57:09 +0100 Subject: [PATCH 381/417] Document None for timeout argument of select.select (#142177) --- Doc/library/select.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst index e821cb01d94..62b5161fb80 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -115,7 +115,7 @@ The module defines the following: :ref:`kevent-objects` below for the methods supported by kevent objects. -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -131,7 +131,7 @@ The module defines the following: platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out in seconds; it may be a non-integer to specify fractions of seconds. - When the *timeout* argument is omitted the function blocks until + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. From d3c888b4ec15dbd7d6b6ef4f15b558af77c228af Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:12:20 +0000 Subject: [PATCH 382/417] gh-139707: Fix example for configure option (GH-142153) Fix example for nre configure option --- Doc/using/configure.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e140ca5d71f..dff0fe036ea 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -339,7 +339,7 @@ General Options .. code-block:: json { - "_gdbm": "The '_gdbm' module is not available in this distribution" + "_gdbm": "The '_gdbm' module is not available in this distribution", "tkinter": "Install the python-tk package to use tkinter", "_tkinter": "Install the python-tk package to use tkinter", } From 8801c6dec79275e9b588ae03e356d8e7656fa3f0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 2 Dec 2025 20:33:40 +0000 Subject: [PATCH 383/417] gh-140677 Add heatmap visualization to Tachyon sampling profiler (#140680) Co-authored-by: Ivona Stojanovic --- Lib/profiling/sampling/__init__.py | 3 +- Lib/profiling/sampling/_css_utils.py | 22 + .../{ => _flamegraph_assets}/flamegraph.css | 326 +---- .../{ => _flamegraph_assets}/flamegraph.js | 0 .../flamegraph_template.html | 0 .../sampling/_heatmap_assets/heatmap.css | 1146 +++++++++++++++++ .../sampling/_heatmap_assets/heatmap.js | 349 +++++ .../sampling/_heatmap_assets/heatmap_index.js | 111 ++ .../heatmap_index_template.html | 118 ++ .../heatmap_pyfile_template.html | 96 ++ .../sampling/_shared_assets/base.css | 369 ++++++ Lib/profiling/sampling/cli.py | 17 +- Lib/profiling/sampling/heatmap_collector.py | 1039 +++++++++++++++ Lib/profiling/sampling/sample.py | 2 +- Lib/profiling/sampling/stack_collector.py | 7 +- Lib/test/test_profiling/test_heatmap.py | 653 ++++++++++ Makefile.pre.in | 3 + ...-10-27-17-00-11.gh-issue-140677.hM9pTq.rst | 4 + 18 files changed, 3939 insertions(+), 326 deletions(-) create mode 100644 Lib/profiling/sampling/_css_utils.py rename Lib/profiling/sampling/{ => _flamegraph_assets}/flamegraph.css (72%) rename Lib/profiling/sampling/{ => _flamegraph_assets}/flamegraph.js (100%) rename Lib/profiling/sampling/{ => _flamegraph_assets}/flamegraph_template.html (100%) create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap.css create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap.js create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_index.js create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html create mode 100644 Lib/profiling/sampling/_shared_assets/base.css create mode 100644 Lib/profiling/sampling/heatmap_collector.py create mode 100644 Lib/test/test_profiling/test_heatmap.py create mode 100644 Misc/NEWS.d/next/Library/2025-10-27-17-00-11.gh-issue-140677.hM9pTq.rst diff --git a/Lib/profiling/sampling/__init__.py b/Lib/profiling/sampling/__init__.py index b493c6aa7eb..6a0bb5e5c2f 100644 --- a/Lib/profiling/sampling/__init__.py +++ b/Lib/profiling/sampling/__init__.py @@ -7,7 +7,8 @@ from .collector import Collector from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector +from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .string_table import StringTable -__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "GeckoCollector", "StringTable") +__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "HeatmapCollector", "GeckoCollector", "StringTable") diff --git a/Lib/profiling/sampling/_css_utils.py b/Lib/profiling/sampling/_css_utils.py new file mode 100644 index 00000000000..40912e9b352 --- /dev/null +++ b/Lib/profiling/sampling/_css_utils.py @@ -0,0 +1,22 @@ +import importlib.resources + + +def get_combined_css(component: str) -> str: + template_dir = importlib.resources.files(__package__) + + base_css = (template_dir / "_shared_assets" / "base.css").read_text(encoding="utf-8") + + if component == "flamegraph": + component_css = ( + template_dir / "_flamegraph_assets" / "flamegraph.css" + ).read_text(encoding="utf-8") + elif component == "heatmap": + component_css = (template_dir / "_heatmap_assets" / "heatmap.css").read_text( + encoding="utf-8" + ) + else: + raise ValueError( + f"Unknown component: {component}. Expected 'flamegraph' or 'heatmap'." + ) + + return f"{base_css}\n\n{component_css}" diff --git a/Lib/profiling/sampling/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css similarity index 72% rename from Lib/profiling/sampling/flamegraph.css rename to Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index 1703815acd9..c75f2324b6d 100644 --- a/Lib/profiling/sampling/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -1,136 +1,20 @@ /* ========================================================================== - Flamegraph Viewer - CSS - Python-branded profiler with dark/light theme support + Flamegraph Viewer - Component-Specific CSS + + DEPENDENCY: Requires _shared_assets/base.css to be loaded first + This file extends the shared foundation with flamegraph-specific styles. ========================================================================== */ /* -------------------------------------------------------------------------- - CSS Variables & Theme System + Layout Overrides (Flamegraph-specific) -------------------------------------------------------------------------- */ -:root { - /* Typography */ - --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", - "Geneva", "Verdana", sans-serif; - --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; - - /* Python brand colors (theme-independent) */ - --python-blue: #3776ab; - --python-blue-light: #4584bb; - --python-blue-lighter: #5592cc; - --python-gold: #ffd43b; - --python-gold-dark: #ffcd02; - --python-gold-light: #ffdc5c; - - /* Heat palette - defined per theme below */ - - /* Layout */ - --sidebar-width: 280px; - --sidebar-collapsed: 44px; - --topbar-height: 52px; - --statusbar-height: 32px; - - /* Transitions */ - --transition-fast: 0.15s ease; - --transition-normal: 0.25s ease; -} - -/* Light theme (default) - Python yellow-to-blue heat palette */ -:root, [data-theme="light"] { - --bg-primary: #ffffff; - --bg-secondary: #f8f9fa; - --bg-tertiary: #e9ecef; - --border: #e9ecef; - --border-subtle: #f0f2f5; - - --text-primary: #2e3338; - --text-secondary: #5a6c7d; - --text-muted: #8b949e; - - --accent: #3776ab; - --accent-hover: #2d5aa0; - --accent-glow: rgba(55, 118, 171, 0.15); - - --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); - --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); - --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); - - --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); - - /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ - --heat-1: #d6e9f8; - --heat-2: #a8d0ef; - --heat-3: #7ba3d1; - --heat-4: #ffe6a8; - --heat-5: #ffd43b; - --heat-6: #ffb84d; - --heat-7: #ff9966; - --heat-8: #ff6347; -} - -/* Dark theme - teal-to-orange heat palette */ -[data-theme="dark"] { - --bg-primary: #0d1117; - --bg-secondary: #161b22; - --bg-tertiary: #21262d; - --border: #30363d; - --border-subtle: #21262d; - - --text-primary: #e6edf3; - --text-secondary: #8b949e; - --text-muted: #6e7681; - - --accent: #58a6ff; - --accent-hover: #79b8ff; - --accent-glow: rgba(88, 166, 255, 0.15); - - --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); - --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); - --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); - - --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); - - /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ - --heat-1: #1e3a5f; - --heat-2: #2d5580; - --heat-3: #4a7ba7; - --heat-4: #5a9fa8; - --heat-5: #7ec488; - --heat-6: #c4de6a; - --heat-7: #f4d44d; - --heat-8: #ff6b35; -} - -/* -------------------------------------------------------------------------- - Base Styles - -------------------------------------------------------------------------- */ - -*, *::before, *::after { - box-sizing: border-box; -} - html, body { - margin: 0; - padding: 0; height: 100%; overflow: hidden; } -body { - font-family: var(--font-sans); - font-size: 14px; - line-height: 1.6; - color: var(--text-primary); - background: var(--bg-primary); - transition: background var(--transition-normal), color var(--transition-normal); -} - -/* -------------------------------------------------------------------------- - Layout Structure - -------------------------------------------------------------------------- */ - .app-layout { - display: flex; - flex-direction: column; height: 100vh; } @@ -141,78 +25,9 @@ .main-content { } /* -------------------------------------------------------------------------- - Top Bar + Search Input (Flamegraph-specific) -------------------------------------------------------------------------- */ -.top-bar { - height: 56px; - background: var(--header-gradient); - display: flex; - align-items: center; - padding: 0 16px; - gap: 16px; - flex-shrink: 0; - box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); - border-bottom: 2px solid var(--python-gold); -} - -/* Brand / Logo */ -.brand { - display: flex; - align-items: center; - gap: 12px; - color: white; - text-decoration: none; - flex-shrink: 0; -} - -.brand-logo { - display: flex; - align-items: center; - justify-content: center; - width: 28px; - height: 28px; - flex-shrink: 0; -} - -/* Style the inlined SVG/img inside brand-logo */ -.brand-logo svg, -.brand-logo img { - width: 28px; - height: 28px; - display: block; - object-fit: contain; - filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); -} - -.brand-info { - display: flex; - flex-direction: column; - line-height: 1.15; -} - -.brand-text { - font-weight: 700; - font-size: 16px; - letter-spacing: -0.3px; - text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); -} - -.brand-subtitle { - font-weight: 500; - font-size: 10px; - opacity: 0.9; - text-transform: uppercase; - letter-spacing: 0.5px; -} - -.brand-divider { - width: 1px; - height: 16px; - background: rgba(255, 255, 255, 0.3); -} - -/* Search */ .search-wrapper { flex: 1; max-width: 360px; @@ -308,39 +123,6 @@ .search-wrapper.has-value .search-clear { display: flex; } -/* Toolbar */ -.toolbar { - display: flex; - align-items: center; - gap: 6px; - margin-left: auto; -} - -.toolbar-btn { - display: flex; - align-items: center; - justify-content: center; - width: 32px; - height: 32px; - padding: 0; - font-size: 15px; - color: white; - background: rgba(255, 255, 255, 0.12); - border: 1px solid rgba(255, 255, 255, 0.18); - border-radius: 6px; - cursor: pointer; - transition: all var(--transition-fast); -} - -.toolbar-btn:hover { - background: rgba(255, 255, 255, 0.22); - border-color: rgba(255, 255, 255, 0.35); -} - -.toolbar-btn:active { - transform: scale(0.95); -} - /* -------------------------------------------------------------------------- Sidebar -------------------------------------------------------------------------- */ @@ -667,11 +449,6 @@ .efficiency-fill::after { animation: shimmer 2s ease-in-out infinite; } -@keyframes shimmer { - 0% { left: -100%; } - 100% { left: 100%; } -} - /* -------------------------------------------------------------------------- Thread Stats Grid (in Sidebar) -------------------------------------------------------------------------- */ @@ -974,56 +751,6 @@ .d3-flame-graph rect.search-dim { opacity: 0.25; } -/* -------------------------------------------------------------------------- - Status Bar - -------------------------------------------------------------------------- */ - -.status-bar { - height: var(--statusbar-height); - background: var(--bg-secondary); - border-top: 1px solid var(--border); - display: flex; - align-items: center; - padding: 0 16px; - gap: 16px; - font-family: var(--font-mono); - font-size: 11px; - color: var(--text-secondary); - flex-shrink: 0; -} - -.status-item { - display: flex; - align-items: center; - gap: 5px; -} - -.status-item::before { - content: ''; - width: 4px; - height: 4px; - background: var(--python-gold); - border-radius: 50%; -} - -.status-item:first-child::before { - display: none; -} - -.status-label { - color: var(--text-muted); -} - -.status-value { - color: var(--text-primary); - font-weight: 500; -} - -.status-value.accent { - color: var(--accent); - font-weight: 600; -} - /* -------------------------------------------------------------------------- Tooltip -------------------------------------------------------------------------- */ @@ -1137,38 +864,7 @@ .tooltip-hint { } /* -------------------------------------------------------------------------- - Animations - -------------------------------------------------------------------------- */ - -@keyframes fadeIn { - from { opacity: 0; } - to { opacity: 1; } -} - -@keyframes slideUp { - from { - opacity: 0; - transform: translateY(12px); - } - to { - opacity: 1; - transform: translateY(0); - } -} - -/* -------------------------------------------------------------------------- - Focus States (Accessibility) - -------------------------------------------------------------------------- */ - -button:focus-visible, -select:focus-visible, -input:focus-visible { - outline: 2px solid var(--python-gold); - outline-offset: 2px; -} - -/* -------------------------------------------------------------------------- - Responsive + Responsive (Flamegraph-specific) -------------------------------------------------------------------------- */ @media (max-width: 900px) { @@ -1185,20 +881,12 @@ @media (max-width: 900px) { width: var(--sidebar-collapsed); } - .brand-subtitle { - display: none; - } - .search-wrapper { max-width: 220px; } } @media (max-width: 600px) { - .toolbar-btn:not(.theme-toggle) { - display: none; - } - .search-wrapper { max-width: 160px; } diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js similarity index 100% rename from Lib/profiling/sampling/flamegraph.js rename to Lib/profiling/sampling/_flamegraph_assets/flamegraph.js diff --git a/Lib/profiling/sampling/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html similarity index 100% rename from Lib/profiling/sampling/flamegraph_template.html rename to Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.css b/Lib/profiling/sampling/_heatmap_assets/heatmap.css new file mode 100644 index 00000000000..44915b2a2da --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.css @@ -0,0 +1,1146 @@ +/* ========================================================================== + Heatmap Viewer - Component-Specific CSS + + DEPENDENCY: Requires _shared_assets/base.css to be loaded first + This file extends the shared foundation with heatmap-specific styles. + ========================================================================== */ + +/* -------------------------------------------------------------------------- + Layout Overrides (Heatmap-specific) + -------------------------------------------------------------------------- */ + +.app-layout { + min-height: 100vh; +} + +/* Sticky top bar for heatmap views */ +.top-bar { + position: sticky; + top: 0; + z-index: 100; +} + +/* Back link in toolbar */ +.back-link { + color: white; + text-decoration: none; + padding: 6px 14px; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + font-size: 13px; + font-weight: 500; + transition: all var(--transition-fast); +} + +.back-link:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +/* -------------------------------------------------------------------------- + Main Content Area + -------------------------------------------------------------------------- */ + +.main-content { + flex: 1; + padding: 24px 3%; + width: 100%; + max-width: 100%; +} + +/* -------------------------------------------------------------------------- + Stats Summary Cards - Enhanced with Icons & Animations + -------------------------------------------------------------------------- */ + +.stats-summary { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 12px; + margin-bottom: 24px; +} + +.stat-card { + display: flex; + align-items: center; + gap: 12px; + background: var(--bg-primary); + border: 2px solid var(--border); + border-radius: 10px; + padding: 14px 16px; + transition: all var(--transition-fast); + animation: slideUp 0.5s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.08s); + position: relative; + overflow: hidden; +} + +.stat-card::before { + content: ''; + position: absolute; + top: 0; + left: 0; + right: 0; + height: 3px; + background: linear-gradient(90deg, var(--python-blue), var(--python-gold)); + opacity: 0; + transition: opacity var(--transition-fast); +} + +.stat-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } +.stat-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } +.stat-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } +.stat-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } +.stat-card:nth-child(5) { --i: 4; --card-color: 220, 53, 69; } +.stat-card:nth-child(6) { --i: 5; --card-color: 23, 162, 184; } + +.stat-card:hover { + border-color: rgba(var(--card-color), 0.6); + background: linear-gradient(135deg, rgba(var(--card-color), 0.08) 0%, var(--bg-primary) 100%); + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(var(--card-color), 0.15); +} + +.stat-card:hover::before { + opacity: 1; +} + +.stat-icon { + width: 40px; + height: 40px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + background: linear-gradient(135deg, rgba(var(--card-color), 0.15) 0%, rgba(var(--card-color), 0.05) 100%); + border: 1px solid rgba(var(--card-color), 0.2); + border-radius: 10px; + flex-shrink: 0; + transition: all var(--transition-fast); +} + +.stat-card:hover .stat-icon { + transform: scale(1.05) rotate(-2deg); + background: linear-gradient(135deg, rgba(var(--card-color), 0.25) 0%, rgba(var(--card-color), 0.1) 100%); +} + +.stat-data { + flex: 1; + min-width: 0; +} + +.stat-value { + font-family: var(--font-mono); + font-size: 1.35em; + font-weight: 800; + color: rgb(var(--card-color)); + display: block; + line-height: 1.1; + letter-spacing: -0.3px; +} + +.stat-label { + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.3px; + margin-top: 2px; +} + +/* Sparkline decoration for stats */ +.stat-sparkline { + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 30px; + opacity: 0.1; + background: linear-gradient(180deg, + transparent 0%, + rgba(var(--card-color), 0.3) 100% + ); + pointer-events: none; +} + +/* -------------------------------------------------------------------------- + Rate Cards (Error Rate, Missed Samples) with Progress Bars + -------------------------------------------------------------------------- */ + +.rate-card { + display: flex; + flex-direction: column; + gap: 12px; + background: var(--bg-primary); + border: 2px solid var(--border); + border-radius: 12px; + padding: 18px 20px; + transition: all var(--transition-fast); + animation: slideUp 0.5s ease-out backwards; + position: relative; + overflow: hidden; +} + +.rate-card:nth-child(5) { animation-delay: 0.32s; --rate-color: 220, 53, 69; } +.rate-card:nth-child(6) { animation-delay: 0.40s; --rate-color: 255, 152, 0; } + +.rate-card:hover { + border-color: rgba(var(--rate-color), 0.5); + transform: translateY(-2px); + box-shadow: 0 6px 20px rgba(var(--rate-color), 0.15); +} + +.rate-header { + display: flex; + justify-content: space-between; + align-items: center; +} + +.rate-info { + display: flex; + align-items: center; + gap: 10px; +} + +.rate-icon { + width: 36px; + height: 36px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + background: linear-gradient(135deg, rgba(var(--rate-color), 0.15) 0%, rgba(var(--rate-color), 0.05) 100%); + border: 1px solid rgba(var(--rate-color), 0.2); + border-radius: 10px; + flex-shrink: 0; +} + +.rate-label { + font-size: 12px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.3px; +} + +.rate-value { + font-family: var(--font-mono); + font-size: 1.4em; + font-weight: 800; + color: rgb(var(--rate-color)); +} + +.rate-bar { + height: 8px; + background: var(--bg-tertiary); + border-radius: 4px; + overflow: hidden; + position: relative; +} + +.rate-fill { + height: 100%; + border-radius: 4px; + transition: width 0.8s ease-out; + position: relative; + overflow: hidden; +} + +.rate-fill.error { + background: linear-gradient(90deg, #dc3545 0%, #ff6b6b 100%); +} + +.rate-fill.warning { + background: linear-gradient(90deg, #ff9800 0%, #ffc107 100%); +} + +.rate-fill.good { + background: linear-gradient(90deg, #28a745 0%, #20c997 100%); +} + +/* Shimmer animation on rate bars */ +.rate-fill::after { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.4) 50%, + transparent 100% + ); + animation: shimmer 2.5s ease-in-out infinite; +} + +/* -------------------------------------------------------------------------- + Section Headers + -------------------------------------------------------------------------- */ + +.section-header { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 16px; + padding-bottom: 12px; + border-bottom: 2px solid var(--python-gold); +} + +.section-title { + font-size: 18px; + font-weight: 700; + color: var(--text-primary); + margin: 0; + flex: 1; +} + +/* -------------------------------------------------------------------------- + Filter Controls + -------------------------------------------------------------------------- */ + +.filter-controls { + display: flex; + gap: 8px; + flex-wrap: wrap; + align-items: center; + margin-bottom: 16px; +} + +.control-btn { + padding: 8px 16px; + background: var(--bg-secondary); + color: var(--text-primary); + border: 1px solid var(--border); + border-radius: 6px; + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: all var(--transition-fast); +} + +.control-btn:hover { + background: var(--accent); + color: white; + border-color: var(--accent); +} + +/* -------------------------------------------------------------------------- + Type Sections (stdlib, project, etc) + -------------------------------------------------------------------------- */ + +.type-section { + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; + margin-bottom: 12px; +} + +.type-header { + padding: 12px 16px; + background: var(--header-gradient); + color: white; + cursor: pointer; + display: flex; + align-items: center; + gap: 10px; + user-select: none; + transition: all var(--transition-fast); + font-weight: 600; +} + +.type-header:hover { + opacity: 0.95; +} + +.type-icon { + font-size: 12px; + transition: transform var(--transition-fast); + min-width: 12px; +} + +.type-title { + font-size: 14px; + flex: 1; +} + +.type-stats { + font-size: 12px; + opacity: 0.9; + background: rgba(255, 255, 255, 0.15); + padding: 4px 10px; + border-radius: 4px; + font-family: var(--font-mono); +} + +.type-content { + padding: 12px; +} + +/* -------------------------------------------------------------------------- + Folder Nodes (hierarchical structure) + -------------------------------------------------------------------------- */ + +.folder-node { + margin-bottom: 6px; +} + +.folder-header { + padding: 8px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: 6px; + cursor: pointer; + display: flex; + align-items: center; + gap: 8px; + user-select: none; + transition: all var(--transition-fast); +} + +.folder-header:hover { + background: var(--accent-glow); + border-color: var(--accent); +} + +.folder-icon { + font-size: 10px; + color: var(--accent); + transition: transform var(--transition-fast); + min-width: 12px; +} + +.folder-name { + flex: 1; + font-weight: 500; + color: var(--text-primary); + font-size: 13px; +} + +.folder-stats { + font-size: 11px; + color: var(--text-secondary); + background: var(--bg-tertiary); + padding: 2px 8px; + border-radius: 4px; + font-family: var(--font-mono); +} + +.folder-content { + padding-left: 20px; + margin-top: 6px; +} + +/* -------------------------------------------------------------------------- + File Items + -------------------------------------------------------------------------- */ + +.files-list { + display: flex; + flex-direction: column; + gap: 4px; + margin-top: 8px; +} + +.file-item { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 12px; + background: var(--bg-primary); + border: 1px solid var(--border-subtle); + border-radius: 6px; + transition: all var(--transition-fast); +} + +.file-item:hover { + background: var(--bg-secondary); + border-color: var(--border); +} + +.file-item .file-link { + flex: 1; + min-width: 0; + font-size: 13px; +} + +.file-samples { + font-size: 12px; + color: var(--text-secondary); + font-weight: 600; + white-space: nowrap; + width: 130px; + flex-shrink: 0; + text-align: right; + font-family: var(--font-mono); +} + +.heatmap-bar-container { + width: 120px; + flex-shrink: 0; + display: flex; + align-items: center; +} + +.heatmap-bar { + flex-shrink: 0; + border-radius: 2px; +} + +/* Links */ +.file-link { + color: var(--accent); + text-decoration: none; + font-weight: 500; + transition: color var(--transition-fast); +} + +.file-link:hover { + color: var(--accent-hover); + text-decoration: underline; +} + +/* -------------------------------------------------------------------------- + Module Badges + -------------------------------------------------------------------------- */ + +.module-badge { + display: inline-block; + padding: 3px 8px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.badge-stdlib { + background: rgba(40, 167, 69, 0.15); + color: #28a745; +} + +.badge-site-packages { + background: rgba(0, 123, 255, 0.15); + color: #007bff; +} + +.badge-project { + background: rgba(255, 193, 7, 0.2); + color: #d39e00; +} + +.badge-other { + background: var(--bg-tertiary); + color: var(--text-secondary); +} + +[data-theme="dark"] .badge-stdlib { + background: rgba(40, 167, 69, 0.25); + color: #5dd879; +} + +[data-theme="dark"] .badge-site-packages { + background: rgba(88, 166, 255, 0.25); + color: #79b8ff; +} + +[data-theme="dark"] .badge-project { + background: rgba(255, 212, 59, 0.25); + color: #ffd43b; +} + +/* ========================================================================== + FILE VIEW STYLES (Code Display) + ========================================================================== */ + +.code-view { + font-family: var(--font-mono); + min-height: 100vh; +} + +/* Code Header (Top Bar for file view) */ +.code-header { + height: var(--topbar-height); + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); + position: sticky; + top: 0; + z-index: 100; +} + +.code-header-content { + display: flex; + align-items: center; + justify-content: space-between; + width: 94%; + max-width: 100%; + margin: 0 auto; +} + +.code-header h1 { + font-size: 14px; + font-weight: 600; + color: white; + margin: 0; + font-family: var(--font-mono); + display: flex; + align-items: center; + gap: 8px; +} + +/* File Stats Bar */ +.file-stats { + background: var(--bg-secondary); + padding: 16px 24px; + border-bottom: 1px solid var(--border); +} + +.file-stats .stats-grid { + width: 94%; + max-width: 100%; + margin: 0 auto; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); + gap: 12px; +} + +.stat-item { + background: var(--bg-primary); + padding: 12px; + border-radius: 8px; + box-shadow: var(--shadow-sm); + text-align: center; + border: 1px solid var(--border); + transition: all var(--transition-fast); +} + +.stat-item:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-md); + border-color: var(--accent); +} + +.stat-item .stat-value { + font-size: 1.4em; + font-weight: 700; + color: var(--accent); +} + +.stat-item .stat-label { + color: var(--text-muted); + font-size: 10px; + margin-top: 2px; +} + +/* Legend */ +.legend { + background: var(--bg-secondary); + padding: 12px 24px; + border-bottom: 1px solid var(--border); +} + +.legend-content { + width: 94%; + max-width: 100%; + margin: 0 auto; + display: flex; + align-items: center; + gap: 20px; + flex-wrap: wrap; +} + +.legend-title { + font-weight: 600; + color: var(--text-primary); + font-size: 13px; + font-family: var(--font-sans); +} + +.legend-gradient { + flex: 1; + max-width: 300px; + height: 24px; + background: linear-gradient(90deg, + var(--bg-tertiary) 0%, + var(--heat-2) 25%, + var(--heat-4) 50%, + var(--heat-6) 75%, + var(--heat-8) 100% + ); + border-radius: 4px; + border: 1px solid var(--border); +} + +.legend-labels { + display: flex; + gap: 12px; + font-size: 11px; + color: var(--text-muted); + font-family: var(--font-sans); +} + +/* Toggle Switch Styles */ +.toggle-switch { + display: inline-flex; + align-items: center; + gap: 8px; + cursor: pointer; + user-select: none; + font-family: var(--font-sans); + transition: opacity var(--transition-fast); +} + +.toggle-switch:hover { + opacity: 0.85; +} + +.toggle-switch .toggle-label { + font-size: 11px; + font-weight: 500; + color: var(--text-muted); + min-width: 55px; + text-align: right; + transition: color var(--transition-fast); +} + +.toggle-switch .toggle-label:last-child { + text-align: left; +} + +.toggle-switch .toggle-label.active { + color: var(--text-primary); + font-weight: 600; +} + +.toggle-track { + position: relative; + width: 36px; + height: 20px; + background: var(--bg-tertiary); + border: 2px solid var(--border); + border-radius: 12px; + transition: all var(--transition-fast); + box-shadow: inset var(--shadow-sm); +} + +.toggle-track:hover { + border-color: var(--text-muted); +} + +.toggle-track.on { + background: var(--accent); + border-color: var(--accent); + box-shadow: 0 0 8px var(--accent-glow); +} + +.toggle-track::after { + content: ''; + position: absolute; + top: 1px; + left: 1px; + width: 14px; + height: 14px; + background: white; + border-radius: 50%; + box-shadow: var(--shadow-sm); + transition: all var(--transition-fast); +} + +.toggle-track.on::after { + transform: translateX(16px); + box-shadow: var(--shadow-md); +} + +/* Specific toggle overrides */ +#toggle-color-mode .toggle-track.on { + background: #8e44ad; + border-color: #8e44ad; + box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); +} + +#toggle-cold .toggle-track.on { + background: #e67e22; + border-color: #e67e22; + box-shadow: 0 0 8px rgba(230, 126, 34, 0.3); +} + +/* Code Container */ +.code-container { + width: 94%; + max-width: 100%; + margin: 16px auto; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px 8px 8px 8px; + box-shadow: var(--shadow-sm); + /* Allow horizontal scroll for long lines, but don't clip sticky header */ +} + +/* Code Header Row */ +.code-header-row { + position: sticky; + top: var(--topbar-height); + z-index: 50; + display: flex; + background: var(--bg-secondary); + border-bottom: 2px solid var(--border); + font-weight: 700; + font-size: 11px; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.5px; + border-radius: 8px 8px 0 0; +} + +.header-line-number { + flex-shrink: 0; + width: 60px; + padding: 8px 10px; + text-align: right; + border-right: 1px solid var(--border); +} + +.header-samples-self, +.header-samples-cumulative { + flex-shrink: 0; + width: 90px; + padding: 8px 10px; + text-align: right; + border-right: 1px solid var(--border); +} + +.header-samples-self { + color: var(--heat-8); +} + +.header-samples-cumulative { + color: var(--accent); +} + +.header-content { + flex: 1; + padding: 8px 15px; +} + +/* Code Lines */ +.code-line { + position: relative; + display: flex; + min-height: 20px; + line-height: 20px; + font-size: 13px; + transition: background var(--transition-fast); + scroll-margin-top: calc(var(--topbar-height) + 50px); +} + +.code-line:hover { + filter: brightness(0.97); +} + +[data-theme="dark"] .code-line:hover { + filter: brightness(1.1); +} + +.line-number { + flex-shrink: 0; + width: 60px; + padding: 0 10px; + text-align: right; + color: var(--text-muted); + background: var(--bg-secondary); + border-right: 1px solid var(--border); + user-select: none; + transition: all var(--transition-fast); +} + +.line-number:hover { + background: var(--accent); + color: white; + cursor: pointer; +} + +.line-samples-self, +.line-samples-cumulative { + flex-shrink: 0; + width: 90px; + padding: 0 10px; + text-align: right; + background: var(--bg-secondary); + border-right: 1px solid var(--border); + font-weight: 600; + user-select: none; + font-size: 12px; +} + +.line-samples-self { + color: var(--heat-8); +} + +.line-samples-cumulative { + color: var(--accent); +} + +.line-content { + flex: 1; + padding: 0 15px; + white-space: pre; + overflow-x: auto; +} + +/* Scrollbar Styling */ +.line-content::-webkit-scrollbar { + height: 6px; +} + +.line-content::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} + +.line-content::-webkit-scrollbar-thumb:hover { + background: var(--text-muted); +} + +/* Navigation Buttons */ +.line-nav-buttons { + position: absolute; + right: 8px; + top: 50%; + transform: translateY(-50%); + display: flex; + gap: 4px; + align-items: center; +} + +.nav-btn { + padding: 2px 6px; + font-size: 12px; + font-weight: 500; + border: 1px solid var(--accent); + border-radius: 4px; + background: var(--bg-primary); + color: var(--accent); + cursor: pointer; + transition: all var(--transition-fast); + user-select: none; + line-height: 1; +} + +.nav-btn:hover:not(:disabled) { + background: var(--accent); + color: white; + transform: translateY(-1px); + box-shadow: var(--shadow-sm); +} + +.nav-btn:active:not(:disabled) { + transform: translateY(0); +} + +.nav-btn:disabled { + opacity: 0.3; + cursor: not-allowed; + color: var(--text-muted); + background: var(--bg-secondary); + border-color: var(--border); +} + +.nav-btn.caller { + color: var(--nav-caller); + border-color: var(--nav-caller); +} + +.nav-btn.callee { + color: var(--nav-callee); + border-color: var(--nav-callee); +} + +.nav-btn.caller:hover:not(:disabled) { + background: var(--nav-caller-hover); + color: white; +} + +.nav-btn.callee:hover:not(:disabled) { + background: var(--nav-callee-hover); + color: white; +} + +/* Highlighted target line */ +.code-line:target { + animation: highlight-line 2s ease-out; +} + +@keyframes highlight-line { + 0% { + background: rgba(255, 212, 59, 0.6) !important; + outline: 3px solid var(--python-gold); + outline-offset: -3px; + } + 50% { + background: rgba(255, 212, 59, 0.5) !important; + outline: 3px solid var(--python-gold); + outline-offset: -3px; + } + 100% { + background: inherit; + outline: 3px solid transparent; + outline-offset: -3px; + } +} + +/* Popup menu for multiple callees */ +.callee-menu { + position: absolute; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: var(--shadow-lg); + padding: 8px; + z-index: 1000; + min-width: 250px; + max-width: 400px; + max-height: 300px; + overflow-y: auto; +} + +.callee-menu-header { + font-weight: 600; + color: var(--text-primary); + margin-bottom: 8px; + padding-bottom: 8px; + border-bottom: 1px solid var(--border); + font-size: 13px; + font-family: var(--font-sans); +} + +.callee-menu-item { + padding: 8px; + margin: 4px 0; + border-radius: 6px; + cursor: pointer; + transition: background var(--transition-fast); + display: flex; + flex-direction: column; + gap: 4px; +} + +.callee-menu-item:hover { + background: var(--bg-secondary); +} + +.callee-menu-func { + font-weight: 500; + color: var(--accent); + font-size: 12px; +} + +.callee-menu-file { + font-size: 11px; + color: var(--text-muted); +} + +.count-badge { + display: inline-block; + background: var(--accent); + color: white; + font-size: 10px; + padding: 2px 6px; + border-radius: 4px; + font-weight: 600; + margin-left: 6px; +} + +/* Callee menu scrollbar */ +.callee-menu::-webkit-scrollbar { + width: 6px; +} + +.callee-menu::-webkit-scrollbar-track { + background: var(--bg-secondary); + border-radius: 3px; +} + +.callee-menu::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} + +/* -------------------------------------------------------------------------- + Scroll Minimap Marker + -------------------------------------------------------------------------- */ + +#scroll_marker { + position: fixed; + z-index: 1000; + right: 0; + top: 0; + width: 12px; + height: 100%; + background: var(--bg-secondary); + border-left: 1px solid var(--border); + pointer-events: none; +} + +#scroll_marker .marker { + position: absolute; + min-height: 3px; + width: 100%; + pointer-events: none; +} + +#scroll_marker .marker.cold { + background: var(--heat-2); +} + +#scroll_marker .marker.warm { + background: var(--heat-5); +} + +#scroll_marker .marker.hot { + background: var(--heat-7); +} + +#scroll_marker .marker.vhot { + background: var(--heat-8); +} + +/* -------------------------------------------------------------------------- + Responsive (Heatmap-specific) + -------------------------------------------------------------------------- */ + +@media (max-width: 1100px) { + .stats-summary { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 900px) { + .main-content { + padding: 16px; + } +} + +@media (max-width: 600px) { + .stats-summary { + grid-template-columns: 1fr; + } + + .file-stats .stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .legend-content { + flex-direction: column; + gap: 12px; + } + + .legend-gradient { + width: 100%; + max-width: none; + } +} diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js new file mode 100644 index 00000000000..ccf82386363 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -0,0 +1,349 @@ +// Tachyon Profiler - Heatmap JavaScript +// Interactive features for the heatmap visualization +// Aligned with Flamegraph viewer design patterns + +// ============================================================================ +// State Management +// ============================================================================ + +let currentMenu = null; +let colorMode = 'self'; // 'self' or 'cumulative' - default to self +let coldCodeHidden = false; + +// ============================================================================ +// Theme Support +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('heatmap-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } + + // Rebuild scroll marker with new theme colors + buildScrollMarker(); +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('heatmap-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } +} + +// ============================================================================ +// Utility Functions +// ============================================================================ + +function createElement(tag, className, textContent = '') { + const el = document.createElement(tag); + if (className) el.className = className; + if (textContent) el.textContent = textContent; + return el; +} + +function calculateMenuPosition(buttonRect, menuWidth, menuHeight) { + const viewport = { width: window.innerWidth, height: window.innerHeight }; + const scroll = { + x: window.pageXOffset || document.documentElement.scrollLeft, + y: window.pageYOffset || document.documentElement.scrollTop + }; + + const left = buttonRect.right + menuWidth + 10 < viewport.width + ? buttonRect.right + scroll.x + 10 + : Math.max(scroll.x + 10, buttonRect.left + scroll.x - menuWidth - 10); + + const top = buttonRect.bottom + menuHeight + 10 < viewport.height + ? buttonRect.bottom + scroll.y + 5 + : Math.max(scroll.y + 10, buttonRect.top + scroll.y - menuHeight - 10); + + return { left, top }; +} + +// ============================================================================ +// Menu Management +// ============================================================================ + +function closeMenu() { + if (currentMenu) { + currentMenu.remove(); + currentMenu = null; + } +} + +function showNavigationMenu(button, items, title) { + closeMenu(); + + const menu = createElement('div', 'callee-menu'); + menu.appendChild(createElement('div', 'callee-menu-header', title)); + + items.forEach(linkData => { + const item = createElement('div', 'callee-menu-item'); + + const funcDiv = createElement('div', 'callee-menu-func'); + funcDiv.textContent = linkData.func; + + if (linkData.count !== undefined && linkData.count > 0) { + const countBadge = createElement('span', 'count-badge'); + countBadge.textContent = linkData.count.toLocaleString(); + countBadge.title = `${linkData.count.toLocaleString()} samples`; + funcDiv.appendChild(document.createTextNode(' ')); + funcDiv.appendChild(countBadge); + } + + item.appendChild(funcDiv); + item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); + item.addEventListener('click', () => window.location.href = linkData.link); + menu.appendChild(item); + }); + + const pos = calculateMenuPosition(button.getBoundingClientRect(), 350, 300); + menu.style.left = `${pos.left}px`; + menu.style.top = `${pos.top}px`; + + document.body.appendChild(menu); + currentMenu = menu; +} + +// ============================================================================ +// Navigation +// ============================================================================ + +function handleNavigationClick(button, e) { + e.stopPropagation(); + + const navData = button.getAttribute('data-nav'); + if (navData) { + window.location.href = JSON.parse(navData).link; + return; + } + + const navMulti = button.getAttribute('data-nav-multi'); + if (navMulti) { + const items = JSON.parse(navMulti); + const title = button.classList.contains('caller') ? 'Choose a caller:' : 'Choose a callee:'; + showNavigationMenu(button, items, title); + } +} + +function scrollToTargetLine() { + if (!window.location.hash) return; + const target = document.querySelector(window.location.hash); + if (target) { + target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + } +} + +// ============================================================================ +// Sample Count & Intensity +// ============================================================================ + +function getSampleCount(line) { + let text; + if (colorMode === 'self') { + text = line.querySelector('.line-samples-self')?.textContent.trim().replace(/,/g, ''); + } else { + text = line.querySelector('.line-samples-cumulative')?.textContent.trim().replace(/,/g, ''); + } + return parseInt(text) || 0; +} + +function getIntensityClass(ratio) { + if (ratio > 0.75) return 'vhot'; + if (ratio > 0.5) return 'hot'; + if (ratio > 0.25) return 'warm'; + return 'cold'; +} + +// ============================================================================ +// Scroll Minimap +// ============================================================================ + +function buildScrollMarker() { + const existing = document.getElementById('scroll_marker'); + if (existing) existing.remove(); + + if (document.body.scrollHeight <= window.innerHeight) return; + + const allLines = document.querySelectorAll('.code-line'); + const lines = Array.from(allLines).filter(line => line.style.display !== 'none'); + const markerScale = window.innerHeight / document.body.scrollHeight; + const lineHeight = Math.min(Math.max(3, window.innerHeight / lines.length), 10); + const maxSamples = Math.max(...Array.from(lines, getSampleCount)); + + const scrollMarker = createElement('div', ''); + scrollMarker.id = 'scroll_marker'; + + let prevLine = -99, lastMark, lastTop; + + lines.forEach((line, index) => { + const samples = getSampleCount(line); + if (samples === 0) return; + + const lineTop = Math.floor(line.offsetTop * markerScale); + const lineNumber = index + 1; + const intensityClass = maxSamples > 0 ? getIntensityClass(samples / maxSamples) : 'cold'; + + if (lineNumber === prevLine + 1 && lastMark?.classList.contains(intensityClass)) { + lastMark.style.height = `${lineTop + lineHeight - lastTop}px`; + } else { + lastMark = createElement('div', `marker ${intensityClass}`); + lastMark.style.height = `${lineHeight}px`; + lastMark.style.top = `${lineTop}px`; + scrollMarker.appendChild(lastMark); + lastTop = lineTop; + } + + prevLine = lineNumber; + }); + + document.body.appendChild(scrollMarker); +} + +// ============================================================================ +// Toggle Controls +// ============================================================================ + +function updateToggleUI(toggleId, isOn) { + const toggle = document.getElementById(toggleId); + if (toggle) { + const track = toggle.querySelector('.toggle-track'); + const labels = toggle.querySelectorAll('.toggle-label'); + if (isOn) { + track.classList.add('on'); + labels[0].classList.remove('active'); + labels[1].classList.add('active'); + } else { + track.classList.remove('on'); + labels[0].classList.add('active'); + labels[1].classList.remove('active'); + } + } +} + +function toggleColdCode() { + coldCodeHidden = !coldCodeHidden; + applyHotFilter(); + updateToggleUI('toggle-cold', coldCodeHidden); + buildScrollMarker(); +} + +function applyHotFilter() { + const lines = document.querySelectorAll('.code-line'); + + lines.forEach(line => { + const selfSamples = line.querySelector('.line-samples-self')?.textContent.trim(); + const cumulativeSamples = line.querySelector('.line-samples-cumulative')?.textContent.trim(); + + let isCold; + if (colorMode === 'self') { + isCold = !selfSamples || selfSamples === ''; + } else { + isCold = !cumulativeSamples || cumulativeSamples === ''; + } + + if (isCold) { + line.style.display = coldCodeHidden ? 'none' : 'flex'; + } else { + line.style.display = 'flex'; + } + }); +} + +function toggleColorMode() { + colorMode = colorMode === 'self' ? 'cumulative' : 'self'; + const lines = document.querySelectorAll('.code-line'); + + lines.forEach(line => { + let bgColor; + if (colorMode === 'self') { + bgColor = line.getAttribute('data-self-color'); + } else { + bgColor = line.getAttribute('data-cumulative-color'); + } + + if (bgColor) { + line.style.background = bgColor; + } + }); + + updateToggleUI('toggle-color-mode', colorMode === 'cumulative'); + + if (coldCodeHidden) { + applyHotFilter(); + } + + buildScrollMarker(); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +document.addEventListener('DOMContentLoaded', function() { + // Restore UI state (theme, etc.) + restoreUIState(); + + // Apply background colors + document.querySelectorAll('.code-line[data-bg-color]').forEach(line => { + const bgColor = line.getAttribute('data-bg-color'); + if (bgColor) { + line.style.background = bgColor; + } + }); + + // Initialize navigation buttons + document.querySelectorAll('.nav-btn').forEach(button => { + button.addEventListener('click', e => handleNavigationClick(button, e)); + }); + + // Initialize line number permalink handlers + document.querySelectorAll('.line-number').forEach(lineNum => { + lineNum.style.cursor = 'pointer'; + lineNum.addEventListener('click', e => { + window.location.hash = `line-${e.target.textContent.trim()}`; + }); + }); + + // Initialize toggle buttons + const toggleColdBtn = document.getElementById('toggle-cold'); + if (toggleColdBtn) { + toggleColdBtn.addEventListener('click', toggleColdCode); + } + + const colorModeBtn = document.getElementById('toggle-color-mode'); + if (colorModeBtn) { + colorModeBtn.addEventListener('click', toggleColorMode); + } + + // Build scroll marker + setTimeout(buildScrollMarker, 200); + + // Setup scroll-to-line behavior + setTimeout(scrollToTargetLine, 100); +}); + +// Close menu when clicking outside +document.addEventListener('click', e => { + if (currentMenu && !currentMenu.contains(e.target) && !e.target.classList.contains('nav-btn')) { + closeMenu(); + } +}); + +// Handle hash changes +window.addEventListener('hashchange', () => setTimeout(scrollToTargetLine, 50)); + +// Rebuild scroll marker on resize +window.addEventListener('resize', buildScrollMarker); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js new file mode 100644 index 00000000000..5f3e65c3310 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js @@ -0,0 +1,111 @@ +// Tachyon Profiler - Heatmap Index JavaScript +// Index page specific functionality + +// ============================================================================ +// Theme Support +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('heatmap-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('heatmap-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } +} + +// ============================================================================ +// Type Section Toggle (stdlib, project, etc) +// ============================================================================ + +function toggleTypeSection(header) { + const section = header.parentElement; + const content = section.querySelector('.type-content'); + const icon = header.querySelector('.type-icon'); + + if (content.style.display === 'none') { + content.style.display = 'block'; + icon.textContent = '\u25BC'; + } else { + content.style.display = 'none'; + icon.textContent = '\u25B6'; + } +} + +// ============================================================================ +// Folder Toggle +// ============================================================================ + +function toggleFolder(header) { + const folder = header.parentElement; + const content = folder.querySelector('.folder-content'); + const icon = header.querySelector('.folder-icon'); + + if (content.style.display === 'none') { + content.style.display = 'block'; + icon.textContent = '\u25BC'; + folder.classList.remove('collapsed'); + } else { + content.style.display = 'none'; + icon.textContent = '\u25B6'; + folder.classList.add('collapsed'); + } +} + +// ============================================================================ +// Expand/Collapse All +// ============================================================================ + +function expandAll() { + // Expand all type sections + document.querySelectorAll('.type-section').forEach(section => { + const content = section.querySelector('.type-content'); + const icon = section.querySelector('.type-icon'); + content.style.display = 'block'; + icon.textContent = '\u25BC'; + }); + + // Expand all folders + document.querySelectorAll('.folder-node').forEach(folder => { + const content = folder.querySelector('.folder-content'); + const icon = folder.querySelector('.folder-icon'); + content.style.display = 'block'; + icon.textContent = '\u25BC'; + folder.classList.remove('collapsed'); + }); +} + +function collapseAll() { + document.querySelectorAll('.folder-node').forEach(folder => { + const content = folder.querySelector('.folder-content'); + const icon = folder.querySelector('.folder-icon'); + content.style.display = 'none'; + icon.textContent = '\u25B6'; + folder.classList.add('collapsed'); + }); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +document.addEventListener('DOMContentLoaded', function() { + restoreUIState(); +}); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html new file mode 100644 index 00000000000..b71bd94c661 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html @@ -0,0 +1,118 @@ + + + + + + Tachyon Profiler - Heatmap Report + + + +
+ +
+
+ Tachyon + + Heatmap Report +
+
+ +
+
+ + +
+ +
+
+
📄
+
+ + Files Profiled +
+
+
+
+
📊
+
+ + Total Snapshots +
+
+
+
+
⏱
+
+ + Duration +
+
+
+
+
⚡
+
+ + Samples/sec +
+
+
+
+
+
+
⚠
+ Error Rate +
+ +
+
+
+
+
+
+
+
+
💥
+ Missed Samples +
+ +
+
+
+
+
+
+ + +
+

Profiled Files

+
+ +
+ + +
+ +
+ +
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + +
+
+ + + + diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html new file mode 100644 index 00000000000..d8b26adfb02 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html @@ -0,0 +1,96 @@ + + + + + + <!-- FILENAME --> - Heatmap + + + +
+ +
+
+ Tachyon + + +
+
+ Back to Index + +
+
+ + +
+
+
+
+
Self Samples
+
+
+
+
Cumulative
+
+
+
+
Lines Hit
+
+
+
%
+
% of Total
+
+
+
+
Max Self
+
+
+
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + → + Hot +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+ +
+
+ + + + diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css new file mode 100644 index 00000000000..20516913496 --- /dev/null +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -0,0 +1,369 @@ +/* ========================================================================== + Python Profiler - Shared CSS Foundation + Design system shared between Flamegraph and Heatmap viewers + ========================================================================== */ + +/* -------------------------------------------------------------------------- + CSS Variables & Theme System + -------------------------------------------------------------------------- */ + +:root { + /* Typography */ + --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", + "Geneva", "Verdana", sans-serif; + --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; + + /* Python brand colors (theme-independent) */ + --python-blue: #3776ab; + --python-blue-light: #4584bb; + --python-blue-lighter: #5592cc; + --python-gold: #ffd43b; + --python-gold-dark: #ffcd02; + --python-gold-light: #ffdc5c; + + /* Heat palette - defined per theme below */ + + /* Layout */ + --sidebar-width: 280px; + --sidebar-collapsed: 44px; + --topbar-height: 56px; + --statusbar-height: 32px; + + /* Transitions */ + --transition-fast: 0.15s ease; + --transition-normal: 0.25s ease; +} + +/* Light theme (default) */ +:root, [data-theme="light"] { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #e9ecef; + --border: #e9ecef; + --border-subtle: #f0f2f5; + + --text-primary: #2e3338; + --text-secondary: #5a6c7d; + --text-muted: #8b949e; + + --accent: #3776ab; + --accent-hover: #2d5aa0; + --accent-glow: rgba(55, 118, 171, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); + + --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); + + /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ + --heat-1: #d6e9f8; + --heat-2: #a8d0ef; + --heat-3: #7ba3d1; + --heat-4: #ffe6a8; + --heat-5: #ffd43b; + --heat-6: #ffb84d; + --heat-7: #ff9966; + --heat-8: #ff6347; + + /* Code view specific */ + --code-bg: #ffffff; + --code-bg-line: #f8f9fa; + --code-border: #e9ecef; + --code-text: #2e3338; + --code-text-muted: #8b949e; + --code-accent: #3776ab; + + /* Navigation colors */ + --nav-caller: #2563eb; + --nav-caller-hover: #1d4ed8; + --nav-callee: #dc2626; + --nav-callee-hover: #b91c1c; +} + +/* Dark theme */ +[data-theme="dark"] { + --bg-primary: #0d1117; + --bg-secondary: #161b22; + --bg-tertiary: #21262d; + --border: #30363d; + --border-subtle: #21262d; + + --text-primary: #e6edf3; + --text-secondary: #8b949e; + --text-muted: #6e7681; + + --accent: #58a6ff; + --accent-hover: #79b8ff; + --accent-glow: rgba(88, 166, 255, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); + + --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); + + /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ + --heat-1: #1e3a5f; + --heat-2: #2d5580; + --heat-3: #4a7ba7; + --heat-4: #5a9fa8; + --heat-5: #7ec488; + --heat-6: #c4de6a; + --heat-7: #f4d44d; + --heat-8: #ff6b35; + + /* Code view specific - dark mode */ + --code-bg: #0d1117; + --code-bg-line: #161b22; + --code-border: #30363d; + --code-text: #e6edf3; + --code-text-muted: #6e7681; + --code-accent: #58a6ff; + + /* Navigation colors - dark theme friendly */ + --nav-caller: #58a6ff; + --nav-caller-hover: #4184e4; + --nav-callee: #f87171; + --nav-callee-hover: #e53e3e; +} + +/* -------------------------------------------------------------------------- + Base Styles + -------------------------------------------------------------------------- */ + +*, *::before, *::after { + box-sizing: border-box; +} + +html, body { + margin: 0; + padding: 0; +} + +body { + font-family: var(--font-sans); + font-size: 14px; + line-height: 1.6; + color: var(--text-primary); + background: var(--bg-primary); + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + transition: background var(--transition-normal), color var(--transition-normal); +} + +/* -------------------------------------------------------------------------- + Layout Structure + -------------------------------------------------------------------------- */ + +.app-layout { + display: flex; + flex-direction: column; +} + +/* -------------------------------------------------------------------------- + Top Bar + -------------------------------------------------------------------------- */ + +.top-bar { + height: var(--topbar-height); + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + flex-shrink: 0; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); +} + +/* Brand / Logo */ +.brand { + display: flex; + align-items: center; + gap: 12px; + color: white; + text-decoration: none; + flex-shrink: 0; +} + +.brand-logo { + display: flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + flex-shrink: 0; +} + +/* Style the inlined SVG/img inside brand-logo */ +.brand-logo svg, +.brand-logo img { + width: 28px; + height: 28px; + display: block; + object-fit: contain; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); +} + +.brand-info { + display: flex; + flex-direction: column; + line-height: 1.15; +} + +.brand-text { + font-weight: 700; + font-size: 16px; + letter-spacing: -0.3px; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); +} + +.brand-subtitle { + font-weight: 500; + font-size: 10px; + opacity: 0.9; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.brand-divider { + width: 1px; + height: 16px; + background: rgba(255, 255, 255, 0.3); +} + +/* Toolbar */ +.toolbar { + display: flex; + align-items: center; + gap: 6px; + margin-left: auto; +} + +.toolbar-btn { + display: flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + padding: 0; + font-size: 15px; + color: white; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); +} + +.toolbar-btn:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +.toolbar-btn:active { + transform: scale(0.95); +} + +/* -------------------------------------------------------------------------- + Status Bar + -------------------------------------------------------------------------- */ + +.status-bar { + height: var(--statusbar-height); + background: var(--bg-secondary); + border-top: 1px solid var(--border); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-secondary); + flex-shrink: 0; +} + +.status-item { + display: flex; + align-items: center; + gap: 5px; +} + +.status-item::before { + content: ''; + width: 4px; + height: 4px; + background: var(--python-gold); + border-radius: 50%; +} + +.status-item:first-child::before { + display: none; +} + +.status-label { + color: var(--text-muted); +} + +.status-value { + color: var(--text-primary); + font-weight: 500; +} + +.status-value.accent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Animations + -------------------------------------------------------------------------- */ + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes slideUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +@keyframes shimmer { + 0% { left: -100%; } + 100% { left: 100%; } +} + +/* -------------------------------------------------------------------------- + Focus States (Accessibility) + -------------------------------------------------------------------------- */ + +button:focus-visible, +select:focus-visible, +input:focus-visible { + outline: 2px solid var(--python-gold); + outline-offset: 2px; +} + +/* -------------------------------------------------------------------------- + Shared Responsive + -------------------------------------------------------------------------- */ + +@media (max-width: 900px) { + .brand-subtitle { + display: none; + } +} + +@media (max-width: 600px) { + .toolbar-btn:not(.theme-toggle) { + display: none; + } +} diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index aede6a4d3e9..5c0e39d7737 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -9,6 +9,7 @@ from .sample import sample, sample_live from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector, FlamegraphCollector +from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .constants import ( PROFILING_MODE_ALL, @@ -71,6 +72,7 @@ class CustomFormatter( "collapsed": "txt", "flamegraph": "html", "gecko": "json", + "heatmap": "html", } COLLECTOR_MAP = { @@ -78,6 +80,7 @@ class CustomFormatter( "collapsed": CollapsedStackCollector, "flamegraph": FlamegraphCollector, "gecko": GeckoCollector, + "heatmap": HeatmapCollector, } @@ -238,14 +241,21 @@ def _add_format_options(parser): dest="format", help="Generate Gecko format for Firefox Profiler", ) + format_group.add_argument( + "--heatmap", + action="store_const", + const="heatmap", + dest="format", + help="Generate interactive HTML heatmap visualization with line-level sample counts", + ) parser.set_defaults(format="pstats") output_group.add_argument( "-o", "--output", dest="outfile", - help="Save output to a file (default: stdout for pstats, " - "auto-generated filename for other formats)", + help="Output path (default: stdout for pstats, auto-generated for others). " + "For heatmap: directory name (default: heatmap_PID)", ) @@ -327,6 +337,9 @@ def _generate_output_filename(format_type, pid): Generated filename """ extension = FORMAT_EXTENSIONS.get(format_type, "txt") + # For heatmap, use cleaner directory name without extension + if format_type == "heatmap": + return f"heatmap_{pid}" return f"{format_type}.{pid}.{extension}" diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py new file mode 100644 index 00000000000..eb51ce33b28 --- /dev/null +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -0,0 +1,1039 @@ +"""Heatmap collector for Python profiling with line-level execution heat visualization.""" + +import base64 +import collections +import html +import importlib.resources +import json +import os +import platform +import site +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Tuple, Optional, Any + +from ._css_utils import get_combined_css +from .stack_collector import StackTraceCollector + + +# ============================================================================ +# Data Classes +# ============================================================================ + +@dataclass +class FileStats: + """Statistics for a single profiled file.""" + filename: str + module_name: str + module_type: str + total_samples: int + total_self_samples: int + num_lines: int + max_samples: int + max_self_samples: int + percentage: float = 0.0 + + +@dataclass +class TreeNode: + """Node in the hierarchical file tree structure.""" + files: List[FileStats] = field(default_factory=list) + samples: int = 0 + count: int = 0 + children: Dict[str, 'TreeNode'] = field(default_factory=dict) + + +@dataclass +class ColorGradient: + """Configuration for heatmap color gradient calculations.""" + # Color stops thresholds + stop_1: float = 0.2 # Blue to cyan transition + stop_2: float = 0.4 # Cyan to green transition + stop_3: float = 0.6 # Green to yellow transition + stop_4: float = 0.8 # Yellow to orange transition + stop_5: float = 1.0 # Orange to red transition + + # Alpha (opacity) values + alpha_very_cold: float = 0.3 + alpha_cold: float = 0.4 + alpha_medium: float = 0.5 + alpha_warm: float = 0.6 + alpha_hot_base: float = 0.7 + alpha_hot_range: float = 0.15 + + # Gradient multiplier + multiplier: int = 5 + + # Cache for calculated colors + cache: Dict[float, Tuple[int, int, int, float]] = field(default_factory=dict) + + +# ============================================================================ +# Module Path Analysis +# ============================================================================ + +def get_python_path_info(): + """Get information about Python installation paths for module extraction. + + Returns: + dict: Dictionary containing stdlib path, site-packages paths, and sys.path entries. + """ + info = { + 'stdlib': None, + 'site_packages': [], + 'sys_path': [] + } + + # Get standard library path from os module location + try: + if hasattr(os, '__file__') and os.__file__: + info['stdlib'] = Path(os.__file__).parent + except (AttributeError, OSError): + pass # Silently continue if we can't determine stdlib path + + # Get site-packages directories + site_packages = [] + try: + site_packages.extend(Path(p) for p in site.getsitepackages()) + except (AttributeError, OSError): + pass # Continue without site packages if unavailable + + # Get user site-packages + try: + user_site = site.getusersitepackages() + if user_site and Path(user_site).exists(): + site_packages.append(Path(user_site)) + except (AttributeError, OSError): + pass # Continue without user site packages + + info['site_packages'] = site_packages + info['sys_path'] = [Path(p) for p in sys.path if p] + + return info + + +def extract_module_name(filename, path_info): + """Extract Python module name and type from file path. + + Args: + filename: Path to the Python file + path_info: Dictionary from get_python_path_info() + + Returns: + tuple: (module_name, module_type) where module_type is one of: + 'stdlib', 'site-packages', 'project', or 'other' + """ + if not filename: + return ('unknown', 'other') + + try: + file_path = Path(filename) + except (ValueError, OSError): + return (str(filename), 'other') + + # Check if it's in stdlib + if path_info['stdlib'] and _is_subpath(file_path, path_info['stdlib']): + try: + rel_path = file_path.relative_to(path_info['stdlib']) + return (_path_to_module(rel_path), 'stdlib') + except ValueError: + pass + + # Check site-packages + for site_pkg in path_info['site_packages']: + if _is_subpath(file_path, site_pkg): + try: + rel_path = file_path.relative_to(site_pkg) + return (_path_to_module(rel_path), 'site-packages') + except ValueError: + continue + + # Check other sys.path entries (project files) + if not str(file_path).startswith(('<', '[')): # Skip special files + for path_entry in path_info['sys_path']: + if _is_subpath(file_path, path_entry): + try: + rel_path = file_path.relative_to(path_entry) + return (_path_to_module(rel_path), 'project') + except ValueError: + continue + + # Fallback: just use the filename + return (_path_to_module(file_path), 'other') + + +def _is_subpath(file_path, parent_path): + try: + file_path.relative_to(parent_path) + return True + except (ValueError, OSError): + return False + + +def _path_to_module(path): + if isinstance(path, str): + path = Path(path) + + # Remove .py extension + if path.suffix == '.py': + path = path.with_suffix('') + + # Convert path separators to dots + parts = path.parts + + # Handle __init__ files - they represent the package itself + if parts and parts[-1] == '__init__': + parts = parts[:-1] + + return '.'.join(parts) if parts else path.stem + + +# ============================================================================ +# Helper Classes +# ============================================================================ + +class _TemplateLoader: + """Loads and caches HTML/CSS/JS templates for heatmap generation.""" + + def __init__(self): + """Load all templates and assets once.""" + self.index_template = None + self.file_template = None + self.index_css = None + self.index_js = None + self.file_css = None + self.file_js = None + self.logo_html = None + + self._load_templates() + + def _load_templates(self): + """Load all template files from _heatmap_assets.""" + try: + template_dir = importlib.resources.files(__package__) + assets_dir = template_dir / "_heatmap_assets" + + # Load HTML templates + self.index_template = (assets_dir / "heatmap_index_template.html").read_text(encoding="utf-8") + self.file_template = (assets_dir / "heatmap_pyfile_template.html").read_text(encoding="utf-8") + + # Load CSS (same file used for both index and file pages) + css_content = get_combined_css("heatmap") + self.index_css = css_content + self.file_css = css_content + + # Load JS + self.index_js = (assets_dir / "heatmap_index.js").read_text(encoding="utf-8") + self.file_js = (assets_dir / "heatmap.js").read_text(encoding="utf-8") + + # Load Python logo + logo_dir = template_dir / "_assets" + try: + png_path = logo_dir / "python-logo-only.png" + b64_logo = base64.b64encode(png_path.read_bytes()).decode("ascii") + self.logo_html = f'' + except (FileNotFoundError, IOError) as e: + self.logo_html = '
' + print(f"Warning: Could not load Python logo: {e}") + + except (FileNotFoundError, IOError) as e: + raise RuntimeError(f"Failed to load heatmap template files: {e}") from e + + +class _TreeBuilder: + """Builds hierarchical tree structure from file statistics.""" + + @staticmethod + def build_file_tree(file_stats: List[FileStats]) -> Dict[str, TreeNode]: + """Build hierarchical tree grouped by module type, then by module structure. + + Args: + file_stats: List of FileStats objects + + Returns: + Dictionary mapping module types to their tree roots + """ + # Group by module type first + type_groups = {'stdlib': [], 'site-packages': [], 'project': [], 'other': []} + for stat in file_stats: + type_groups[stat.module_type].append(stat) + + # Build tree for each type + trees = {} + for module_type, stats in type_groups.items(): + if not stats: + continue + + root_node = TreeNode() + + for stat in stats: + module_name = stat.module_name + parts = module_name.split('.') + + # Navigate/create tree structure + current_node = root_node + for i, part in enumerate(parts): + if i == len(parts) - 1: + # Last part - store the file + current_node.files.append(stat) + else: + # Intermediate part - create or navigate + if part not in current_node.children: + current_node.children[part] = TreeNode() + current_node = current_node.children[part] + + # Calculate aggregate stats for this type's tree + _TreeBuilder._calculate_node_stats(root_node) + trees[module_type] = root_node + + return trees + + @staticmethod + def _calculate_node_stats(node: TreeNode) -> Tuple[int, int]: + """Recursively calculate aggregate statistics for tree nodes. + + Args: + node: TreeNode to calculate stats for + + Returns: + Tuple of (total_samples, file_count) + """ + total_samples = 0 + file_count = 0 + + # Count files at this level + for file_stat in node.files: + total_samples += file_stat.total_samples + file_count += 1 + + # Recursively process children + for child in node.children.values(): + child_samples, child_count = _TreeBuilder._calculate_node_stats(child) + total_samples += child_samples + file_count += child_count + + node.samples = total_samples + node.count = file_count + return total_samples, file_count + + +class _HtmlRenderer: + """Renders hierarchical tree structures as HTML.""" + + def __init__(self, file_index: Dict[str, str], color_gradient: ColorGradient, + calculate_intensity_color_func): + """Initialize renderer with file index and color calculation function. + + Args: + file_index: Mapping from filenames to HTML file names + color_gradient: ColorGradient configuration + calculate_intensity_color_func: Function to calculate colors + """ + self.file_index = file_index + self.color_gradient = color_gradient + self.calculate_intensity_color = calculate_intensity_color_func + self.heatmap_bar_height = 16 + + def render_hierarchical_html(self, trees: Dict[str, TreeNode]) -> str: + """Build hierarchical HTML with type sections and collapsible module folders. + + Args: + trees: Dictionary mapping module types to tree roots + + Returns: + Complete HTML string for all sections + """ + type_names = { + 'stdlib': '📚 Standard Library', + 'site-packages': '📦 Site Packages', + 'project': '🏗️ Project Files', + 'other': '📄 Other Files' + } + + sections = [] + for module_type in ['project', 'stdlib', 'site-packages', 'other']: + if module_type not in trees: + continue + + tree = trees[module_type] + + # Project starts expanded, others start collapsed + is_collapsed = module_type in {'stdlib', 'site-packages', 'other'} + icon = '▶' if is_collapsed else '▼' + content_style = ' style="display: none;"' if is_collapsed else '' + + section_html = f''' +
+
+ {icon} + {type_names[module_type]} + ({tree.count} files, {tree.samples:,} samples) +
+
+''' + + # Render root folders + root_folders = sorted(tree.children.items(), + key=lambda x: x[1].samples, reverse=True) + + for folder_name, folder_node in root_folders: + section_html += self._render_folder(folder_node, folder_name, level=1) + + # Render root files (files not in any module) + if tree.files: + sorted_files = sorted(tree.files, key=lambda x: x.total_samples, reverse=True) + section_html += '
\n' + for stat in sorted_files: + section_html += self._render_file_item(stat, indent=' ') + section_html += '
\n' + + section_html += '
\n
\n' + sections.append(section_html) + + return '\n'.join(sections) + + def _render_folder(self, node: TreeNode, name: str, level: int = 1) -> str: + """Render a single folder node recursively. + + Args: + node: TreeNode to render + name: Display name for the folder + level: Nesting level for indentation + + Returns: + HTML string for this folder and its contents + """ + indent = ' ' * level + parts = [] + + # Render folder header (collapsed by default) + parts.append(f'{indent}') + + return '\n'.join(parts) + + def _render_file_item(self, stat: FileStats, indent: str = '') -> str: + """Render a single file item with heatmap bar. + + Args: + stat: FileStats object + indent: Indentation string + + Returns: + HTML string for file item + """ + full_path = html.escape(stat.filename) + module_name = html.escape(stat.module_name) + + intensity = stat.percentage / 100.0 + r, g, b, alpha = self.calculate_intensity_color(intensity) + bg_color = f"rgba({r}, {g}, {b}, {alpha})" + bar_width = min(stat.percentage, 100) + + html_file = self.file_index[stat.filename] + + return (f'{indent}
\n' + f'{indent} 📄 {module_name}\n' + f'{indent} {stat.total_samples:,} samples\n' + f'{indent}
\n' + f'{indent}
\n') + + +# ============================================================================ +# Main Collector Class +# ============================================================================ + +class HeatmapCollector(StackTraceCollector): + """Collector that generates coverage.py-style heatmap HTML output with line intensity. + + This collector creates detailed HTML reports showing which lines of code + were executed most frequently during profiling, similar to coverage.py + but showing execution "heat" rather than just coverage. + """ + + # File naming and formatting constants + FILE_INDEX_FORMAT = "file_{:04d}.html" + + def __init__(self, *args, **kwargs): + """Initialize the heatmap collector with data structures for analysis.""" + super().__init__(*args, **kwargs) + + # Sample counting data structures + self.line_samples = collections.Counter() + self.file_samples = collections.defaultdict(collections.Counter) + self.line_self_samples = collections.Counter() + self.file_self_samples = collections.defaultdict(collections.Counter) + + # Call graph data structures for navigation + self.call_graph = collections.defaultdict(list) + self.callers_graph = collections.defaultdict(list) + self.function_definitions = {} + + # Edge counting for call path analysis + self.edge_samples = collections.Counter() + + # Statistics and metadata + self._total_samples = 0 + self._path_info = get_python_path_info() + self.stats = {} + + # Color gradient configuration + self._color_gradient = ColorGradient() + + # Template loader (loads all templates once) + self._template_loader = _TemplateLoader() + + # File index (populated during export) + self.file_index = {} + + @property + def _color_cache(self): + """Compatibility property for accessing color cache.""" + return self._color_gradient.cache + + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, missed_samples=None, **kwargs): + """Set profiling statistics to include in heatmap output. + + Args: + sample_interval_usec: Sampling interval in microseconds + duration_sec: Total profiling duration in seconds + sample_rate: Effective sampling rate + error_rate: Optional error rate during profiling + missed_samples: Optional percentage of missed samples + **kwargs: Additional statistics to include + """ + self.stats = { + "sample_interval_usec": sample_interval_usec, + "duration_sec": duration_sec, + "sample_rate": sample_rate, + "error_rate": error_rate, + "missed_samples": missed_samples, + "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "platform": platform.platform(), + } + self.stats.update(kwargs) + + def process_frames(self, frames, thread_id): + """Process stack frames and count samples per line. + + Args: + frames: List of frame tuples (filename, lineno, funcname) + frames[0] is the leaf (top of stack, where execution is) + thread_id: Thread ID for this stack trace + """ + self._total_samples += 1 + + # Count each line in the stack and build call graph + for i, frame_info in enumerate(frames): + filename, lineno, funcname = frame_info + + if not self._is_valid_frame(filename, lineno): + continue + + # frames[0] is the leaf - where execution is actually happening + is_leaf = (i == 0) + self._record_line_sample(filename, lineno, funcname, is_leaf=is_leaf) + + # Build call graph for adjacent frames + if i + 1 < len(frames): + self._record_call_relationship(frames[i], frames[i + 1]) + + def _is_valid_frame(self, filename, lineno): + """Check if a frame should be included in the heatmap.""" + # Skip internal or invalid files + if not filename or filename.startswith('<') or filename.startswith('['): + return False + + # Skip invalid frames with corrupted filename data + if filename == "__init__" and lineno == 0: + return False + + return True + + def _record_line_sample(self, filename, lineno, funcname, is_leaf=False): + """Record a sample for a specific line.""" + # Track cumulative samples (all occurrences in stack) + self.line_samples[(filename, lineno)] += 1 + self.file_samples[filename][lineno] += 1 + + # Track self/leaf samples (only when at top of stack) + if is_leaf: + self.line_self_samples[(filename, lineno)] += 1 + self.file_self_samples[filename][lineno] += 1 + + # Record function definition location + if funcname and (filename, funcname) not in self.function_definitions: + self.function_definitions[(filename, funcname)] = lineno + + def _record_call_relationship(self, callee_frame, caller_frame): + """Record caller/callee relationship between adjacent frames.""" + callee_filename, callee_lineno, callee_funcname = callee_frame + caller_filename, caller_lineno, caller_funcname = caller_frame + + # Skip internal files for call graph + if callee_filename.startswith('<') or callee_filename.startswith('['): + return + + # Get the callee's function definition line + callee_def_line = self.function_definitions.get( + (callee_filename, callee_funcname), callee_lineno + ) + + # Record caller -> callee relationship + caller_key = (caller_filename, caller_lineno) + callee_info = (callee_filename, callee_def_line, callee_funcname) + if callee_info not in self.call_graph[caller_key]: + self.call_graph[caller_key].append(callee_info) + + # Record callee <- caller relationship + callee_key = (callee_filename, callee_def_line) + caller_info = (caller_filename, caller_lineno, caller_funcname) + if caller_info not in self.callers_graph[callee_key]: + self.callers_graph[callee_key].append(caller_info) + + # Count this call edge for path analysis + edge_key = (caller_key, callee_key) + self.edge_samples[edge_key] += 1 + + def export(self, output_path): + """Export heatmap data as HTML files in a directory. + + Args: + output_path: Path where to create the heatmap output directory + """ + if not self.file_samples: + print("Warning: No heatmap data to export") + return + + try: + output_dir = self._prepare_output_directory(output_path) + file_stats = self._calculate_file_stats() + self._create_file_index(file_stats) + + # Generate individual file reports + self._generate_file_reports(output_dir, file_stats) + + # Generate index page + self._generate_index_html(output_dir / 'index.html', file_stats) + + self._print_export_summary(output_dir, file_stats) + + except Exception as e: + print(f"Error: Failed to export heatmap: {e}") + raise + + def _prepare_output_directory(self, output_path): + """Create output directory for heatmap files.""" + output_dir = Path(output_path) + if output_dir.suffix == '.html': + output_dir = output_dir.with_suffix('') + + try: + output_dir.mkdir(exist_ok=True, parents=True) + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to create output directory {output_dir}: {e}") from e + + return output_dir + + def _create_file_index(self, file_stats: List[FileStats]): + """Create mapping from filenames to HTML file names.""" + self.file_index = { + stat.filename: self.FILE_INDEX_FORMAT.format(i) + for i, stat in enumerate(file_stats) + } + + def _generate_file_reports(self, output_dir, file_stats: List[FileStats]): + """Generate HTML report for each source file.""" + for stat in file_stats: + file_path = output_dir / self.file_index[stat.filename] + line_counts = self.file_samples[stat.filename] + valid_line_counts = {line: count for line, count in line_counts.items() if line >= 0} + + self_counts = self.file_self_samples.get(stat.filename, {}) + valid_self_counts = {line: count for line, count in self_counts.items() if line >= 0} + + self._generate_file_html( + file_path, + stat.filename, + valid_line_counts, + valid_self_counts, + stat + ) + + def _print_export_summary(self, output_dir, file_stats: List[FileStats]): + """Print summary of exported heatmap.""" + print(f"Heatmap output written to {output_dir}/") + print(f" - Index: {output_dir / 'index.html'}") + print(f" - {len(file_stats)} source file(s) analyzed") + + def _calculate_file_stats(self) -> List[FileStats]: + """Calculate statistics for each file. + + Returns: + List of FileStats objects sorted by total samples + """ + file_stats = [] + for filename, line_counts in self.file_samples.items(): + # Skip special frames + if filename in ('~', '...', '.') or filename.startswith('<') or filename.startswith('['): + continue + + # Filter out lines with -1 (special frames) + valid_line_counts = {line: count for line, count in line_counts.items() if line >= 0} + if not valid_line_counts: + continue + + # Get self samples for this file + self_line_counts = self.file_self_samples.get(filename, {}) + valid_self_counts = {line: count for line, count in self_line_counts.items() if line >= 0} + + total_samples = sum(valid_line_counts.values()) + total_self_samples = sum(valid_self_counts.values()) + num_lines = len(valid_line_counts) + max_samples = max(valid_line_counts.values()) + max_self_samples = max(valid_self_counts.values()) if valid_self_counts else 0 + module_name, module_type = extract_module_name(filename, self._path_info) + + file_stats.append(FileStats( + filename=filename, + module_name=module_name, + module_type=module_type, + total_samples=total_samples, + total_self_samples=total_self_samples, + num_lines=num_lines, + max_samples=max_samples, + max_self_samples=max_self_samples, + percentage=0.0 + )) + + # Sort by total samples and calculate percentages + file_stats.sort(key=lambda x: x.total_samples, reverse=True) + if file_stats: + max_total = file_stats[0].total_samples + for stat in file_stats: + stat.percentage = (stat.total_samples / max_total * 100) if max_total > 0 else 0 + + return file_stats + + def _generate_index_html(self, index_path: Path, file_stats: List[FileStats]): + """Generate index.html with list of all profiled files.""" + # Build hierarchical tree + tree = _TreeBuilder.build_file_tree(file_stats) + + # Render tree as HTML + renderer = _HtmlRenderer(self.file_index, self._color_gradient, + self._calculate_intensity_color) + sections_html = renderer.render_hierarchical_html(tree) + + # Format error rate and missed samples with bar classes + error_rate = self.stats.get('error_rate') + if error_rate is not None: + error_rate_str = f"{error_rate:.1f}%" + error_rate_width = min(error_rate, 100) + # Determine bar color class based on rate + if error_rate < 5: + error_rate_class = "good" + elif error_rate < 15: + error_rate_class = "warning" + else: + error_rate_class = "error" + else: + error_rate_str = "N/A" + error_rate_width = 0 + error_rate_class = "good" + + missed_samples = self.stats.get('missed_samples') + if missed_samples is not None: + missed_samples_str = f"{missed_samples:.1f}%" + missed_samples_width = min(missed_samples, 100) + if missed_samples < 5: + missed_samples_class = "good" + elif missed_samples < 15: + missed_samples_class = "warning" + else: + missed_samples_class = "error" + else: + missed_samples_str = "N/A" + missed_samples_width = 0 + missed_samples_class = "good" + + # Populate template + replacements = { + "": f"", + "": f"", + "": self._template_loader.logo_html, + "": str(len(file_stats)), + "": f"{self._total_samples:,}", + "": f"{self.stats.get('duration_sec', 0):.1f}s", + "": f"{self.stats.get('sample_rate', 0):.1f}", + "": error_rate_str, + "": str(error_rate_width), + "": error_rate_class, + "": missed_samples_str, + "": str(missed_samples_width), + "": missed_samples_class, + "": sections_html, + } + + html_content = self._template_loader.index_template + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + try: + index_path.write_text(html_content, encoding='utf-8') + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to write index file {index_path}: {e}") from e + + def _calculate_intensity_color(self, intensity: float) -> Tuple[int, int, int, float]: + """Calculate RGB color and alpha for given intensity (0-1 range). + + Returns (r, g, b, alpha) tuple representing the heatmap color gradient: + blue -> green -> yellow -> orange -> red + + Results are cached to improve performance. + """ + # Round to 3 decimal places for cache key + cache_key = round(intensity, 3) + if cache_key in self._color_gradient.cache: + return self._color_gradient.cache[cache_key] + + gradient = self._color_gradient + m = gradient.multiplier + + # Color stops with (threshold, rgb_func, alpha_func) + stops = [ + (gradient.stop_1, + lambda i: (0, int(150 * i * m), 255), + lambda i: gradient.alpha_very_cold), + (gradient.stop_2, + lambda i: (0, 255, int(255 * (1 - (i - gradient.stop_1) * m))), + lambda i: gradient.alpha_cold), + (gradient.stop_3, + lambda i: (int(255 * (i - gradient.stop_2) * m), 255, 0), + lambda i: gradient.alpha_medium), + (gradient.stop_4, + lambda i: (255, int(200 - 100 * (i - gradient.stop_3) * m), 0), + lambda i: gradient.alpha_warm), + (gradient.stop_5, + lambda i: (255, int(100 * (1 - (i - gradient.stop_4) * m)), 0), + lambda i: gradient.alpha_hot_base + gradient.alpha_hot_range * (i - gradient.stop_4) * m), + ] + + result = None + for threshold, rgb_func, alpha_func in stops: + if intensity < threshold or threshold == gradient.stop_5: + r, g, b = rgb_func(intensity) + result = (r, g, b, alpha_func(intensity)) + break + + # Fallback + if result is None: + result = (255, 0, 0, 0.75) + + # Cache the result + self._color_gradient.cache[cache_key] = result + return result + + def _generate_file_html(self, output_path: Path, filename: str, + line_counts: Dict[int, int], self_counts: Dict[int, int], + file_stat: FileStats): + """Generate HTML for a single source file with heatmap coloring.""" + # Read source file + try: + source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines() + except (IOError, OSError) as e: + if not (filename.startswith('<') or filename.startswith('[') or + filename in ('~', '...', '.') or len(filename) < 2): + print(f"Warning: Could not read source file {filename}: {e}") + source_lines = [f"# Source file not available: {filename}"] + + # Generate HTML for each line + max_samples = max(line_counts.values()) if line_counts else 1 + max_self_samples = max(self_counts.values()) if self_counts else 1 + code_lines_html = [ + self._build_line_html(line_num, line_content, line_counts, self_counts, + max_samples, max_self_samples, filename) + for line_num, line_content in enumerate(source_lines, start=1) + ] + + # Populate template + replacements = { + "": html.escape(filename), + "": f"{file_stat.total_samples:,}", + "": f"{file_stat.total_self_samples:,}", + "": str(file_stat.num_lines), + "": f"{file_stat.percentage:.2f}", + "": str(file_stat.max_samples), + "": str(file_stat.max_self_samples), + "": ''.join(code_lines_html), + "": f"", + "": f"", + } + + html_content = self._template_loader.file_template + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + try: + output_path.write_text(html_content, encoding='utf-8') + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to write file {output_path}: {e}") from e + + def _build_line_html(self, line_num: int, line_content: str, + line_counts: Dict[int, int], self_counts: Dict[int, int], + max_samples: int, max_self_samples: int, filename: str) -> str: + """Build HTML for a single line of source code.""" + cumulative_samples = line_counts.get(line_num, 0) + self_samples = self_counts.get(line_num, 0) + + # Calculate colors for both self and cumulative modes + if cumulative_samples > 0: + cumulative_intensity = cumulative_samples / max_samples if max_samples > 0 else 0 + self_intensity = self_samples / max_self_samples if max_self_samples > 0 and self_samples > 0 else 0 + + # Default to self-based coloring + intensity = self_intensity if self_samples > 0 else cumulative_intensity + r, g, b, alpha = self._calculate_intensity_color(intensity) + bg_color = f"rgba({r}, {g}, {b}, {alpha})" + + # Pre-calculate colors for both modes (for JS toggle) + self_bg_color = self._format_color_for_intensity(self_intensity) if self_samples > 0 else "transparent" + cumulative_bg_color = self._format_color_for_intensity(cumulative_intensity) + + self_display = f"{self_samples:,}" if self_samples > 0 else "" + cumulative_display = f"{cumulative_samples:,}" + tooltip = f"Self: {self_samples:,}, Total: {cumulative_samples:,}" + else: + bg_color = "transparent" + self_bg_color = "transparent" + cumulative_bg_color = "transparent" + self_display = "" + cumulative_display = "" + tooltip = "" + + # Get navigation buttons + nav_buttons_html = self._build_navigation_buttons(filename, line_num) + + # Build line HTML + line_html = html.escape(line_content.rstrip('\n')) + title_attr = f' title="{html.escape(tooltip)}"' if tooltip else "" + + return ( + f'
\n' + f'
{line_num}
\n' + f'
{self_display}
\n' + f'
{cumulative_display}
\n' + f'
{line_html}
\n' + f' {nav_buttons_html}\n' + f'
\n' + ) + + def _format_color_for_intensity(self, intensity: float) -> str: + """Format color as rgba() string for given intensity.""" + r, g, b, alpha = self._calculate_intensity_color(intensity) + return f"rgba({r}, {g}, {b}, {alpha})" + + def _build_navigation_buttons(self, filename: str, line_num: int) -> str: + """Build navigation buttons for callers/callees.""" + line_key = (filename, line_num) + caller_list = self._deduplicate_by_function(self.callers_graph.get(line_key, [])) + callee_list = self._deduplicate_by_function(self.call_graph.get(line_key, [])) + + # Get edge counts for each caller/callee + callers_with_counts = self._get_edge_counts(line_key, caller_list, is_caller=True) + callees_with_counts = self._get_edge_counts(line_key, callee_list, is_caller=False) + + # Build navigation buttons with counts + caller_btn = self._create_navigation_button(callers_with_counts, 'caller', '▲') + callee_btn = self._create_navigation_button(callees_with_counts, 'callee', '▼') + + if caller_btn or callee_btn: + return f'
{caller_btn}{callee_btn}
' + return '' + + def _get_edge_counts(self, line_key: Tuple[str, int], + items: List[Tuple[str, int, str]], + is_caller: bool) -> List[Tuple[str, int, str, int]]: + """Get sample counts for each caller/callee edge.""" + result = [] + for file, line, func in items: + edge_line_key = (file, line) + if is_caller: + edge_key = (edge_line_key, line_key) + else: + edge_key = (line_key, edge_line_key) + + count = self.edge_samples.get(edge_key, 0) + result.append((file, line, func, count)) + + result.sort(key=lambda x: x[3], reverse=True) + return result + + def _deduplicate_by_function(self, items: List[Tuple[str, int, str]]) -> List[Tuple[str, int, str]]: + """Remove duplicate entries based on (file, function) key.""" + seen = {} + result = [] + for file, line, func in items: + key = (file, func) + if key not in seen: + seen[key] = True + result.append((file, line, func)) + return result + + def _create_navigation_button(self, items_with_counts: List[Tuple[str, int, str, int]], + btn_class: str, arrow: str) -> str: + """Create HTML for a navigation button with sample counts.""" + # Filter valid items + valid_items = [(f, l, fn, cnt) for f, l, fn, cnt in items_with_counts + if f in self.file_index and l > 0] + if not valid_items: + return "" + + if len(valid_items) == 1: + file, line, func, count = valid_items[0] + target_html = self.file_index[file] + nav_data = json.dumps({'link': f"{target_html}#line-{line}", 'func': func}) + title = f"Go to {btn_class}: {html.escape(func)} ({count:,} samples)" + return f'' + + # Multiple items - create menu + total_samples = sum(cnt for _, _, _, cnt in valid_items) + items_data = [ + { + 'file': os.path.basename(file), + 'func': func, + 'count': count, + 'link': f"{self.file_index[file]}#line-{line}" + } + for file, line, func, count in valid_items + ] + items_json = html.escape(json.dumps(items_data)) + title = f"{len(items_data)} {btn_class}s ({total_samples:,} samples)" + return f'' diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 88d9a4fa13b..46fc1a05afa 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -10,6 +10,7 @@ from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector, FlamegraphCollector +from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .constants import ( PROFILING_MODE_WALL, @@ -25,7 +26,6 @@ _FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None - class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True): self.pid = pid diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 146a058a03a..e2653609313 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -6,6 +6,7 @@ import linecache import os +from ._css_utils import get_combined_css from .collector import Collector from .string_table import StringTable @@ -331,9 +332,9 @@ def _create_flamegraph_html(self, data): fg_js_path = d3_flame_graph_dir / "d3-flamegraph.min.js" fg_tooltip_js_path = d3_flame_graph_dir / "d3-flamegraph-tooltip.min.js" - html_template = (template_dir / "flamegraph_template.html").read_text(encoding="utf-8") - css_content = (template_dir / "flamegraph.css").read_text(encoding="utf-8") - js_content = (template_dir / "flamegraph.js").read_text(encoding="utf-8") + html_template = (template_dir / "_flamegraph_assets" / "flamegraph_template.html").read_text(encoding="utf-8") + css_content = get_combined_css("flamegraph") + js_content = (template_dir / "_flamegraph_assets" / "flamegraph.js").read_text(encoding="utf-8") # Inline first-party CSS/JS html_template = html_template.replace( diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py new file mode 100644 index 00000000000..a6ff3b83ea1 --- /dev/null +++ b/Lib/test/test_profiling/test_heatmap.py @@ -0,0 +1,653 @@ +"""Tests for the heatmap collector (profiling.sampling).""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from profiling.sampling.heatmap_collector import ( + HeatmapCollector, + get_python_path_info, + extract_module_name, +) + +from test.support import captured_stdout, captured_stderr + + +# ============================================================================= +# Unit Tests for Public Helper Functions +# ============================================================================= + +class TestPathInfoFunctions(unittest.TestCase): + """Test public helper functions for path information.""" + + def test_get_python_path_info_returns_dict(self): + """Test that get_python_path_info returns a dictionary with expected keys.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info, dict) + self.assertIn('stdlib', path_info) + self.assertIn('site_packages', path_info) + self.assertIn('sys_path', path_info) + + def test_get_python_path_info_stdlib_is_path_or_none(self): + """Test that stdlib is either a Path object or None.""" + path_info = get_python_path_info() + + if path_info['stdlib'] is not None: + self.assertIsInstance(path_info['stdlib'], Path) + + def test_get_python_path_info_site_packages_is_list(self): + """Test that site_packages is a list.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info['site_packages'], list) + for item in path_info['site_packages']: + self.assertIsInstance(item, Path) + + def test_get_python_path_info_sys_path_is_list(self): + """Test that sys_path is a list of Path objects.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info['sys_path'], list) + for item in path_info['sys_path']: + self.assertIsInstance(item, Path) + + def test_extract_module_name_with_none(self): + """Test extract_module_name with None filename.""" + path_info = get_python_path_info() + module_name, module_type = extract_module_name(None, path_info) + + self.assertEqual(module_name, 'unknown') + self.assertEqual(module_type, 'other') + + def test_extract_module_name_with_empty_string(self): + """Test extract_module_name with empty filename.""" + path_info = get_python_path_info() + module_name, module_type = extract_module_name('', path_info) + + self.assertEqual(module_name, 'unknown') + self.assertEqual(module_type, 'other') + + def test_extract_module_name_with_stdlib_file(self): + """Test extract_module_name with a standard library file.""" + path_info = get_python_path_info() + + # Use os module as a known stdlib file + if path_info['stdlib']: + stdlib_file = str(path_info['stdlib'] / 'os.py') + module_name, module_type = extract_module_name(stdlib_file, path_info) + + self.assertEqual(module_type, 'stdlib') + self.assertIn('os', module_name) + + def test_extract_module_name_with_project_file(self): + """Test extract_module_name with a project file.""" + path_info = get_python_path_info() + + # Create a mock project file path + if path_info['sys_path']: + # Use current directory as project path + project_file = '/some/project/path/mymodule.py' + module_name, module_type = extract_module_name(project_file, path_info) + + # Should classify as 'other' if not in sys.path + self.assertIn(module_type, ['project', 'other']) + + def test_extract_module_name_removes_py_extension(self): + """Test that .py extension is removed from module names.""" + path_info = get_python_path_info() + + # Test with a simple .py file + module_name, module_type = extract_module_name('/path/to/test.py', path_info) + + # Module name should not contain .py + self.assertNotIn('.py', module_name) + + def test_extract_module_name_with_special_files(self): + """Test extract_module_name with special filenames like .""" + path_info = get_python_path_info() + + special_files = ['', '', '[eval]'] + for special_file in special_files: + module_name, module_type = extract_module_name(special_file, path_info) + self.assertEqual(module_type, 'other') + + +# ============================================================================= +# Unit Tests for HeatmapCollector Public API +# ============================================================================= + +class TestHeatmapCollectorInit(unittest.TestCase): + """Test HeatmapCollector initialization.""" + + def test_init_creates_empty_data_structures(self): + """Test that __init__ creates empty data structures.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Check that data structures are initialized + self.assertIsInstance(collector.line_samples, dict) + self.assertIsInstance(collector.file_samples, dict) + self.assertIsInstance(collector.line_self_samples, dict) + self.assertIsInstance(collector.file_self_samples, dict) + self.assertIsInstance(collector.call_graph, dict) + self.assertIsInstance(collector.callers_graph, dict) + self.assertIsInstance(collector.function_definitions, dict) + self.assertIsInstance(collector.edge_samples, dict) + + # Check that they're empty + self.assertEqual(len(collector.line_samples), 0) + self.assertEqual(len(collector.file_samples), 0) + self.assertEqual(len(collector.line_self_samples), 0) + self.assertEqual(len(collector.file_self_samples), 0) + + def test_init_sets_total_samples_to_zero(self): + """Test that total samples starts at zero.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertEqual(collector._total_samples, 0) + + def test_init_creates_color_cache(self): + """Test that color cache is initialized.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertIsInstance(collector._color_cache, dict) + self.assertEqual(len(collector._color_cache), 0) + + def test_init_gets_path_info(self): + """Test that path info is retrieved during init.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertIsNotNone(collector._path_info) + self.assertIn('stdlib', collector._path_info) + + +class TestHeatmapCollectorSetStats(unittest.TestCase): + """Test HeatmapCollector.set_stats() method.""" + + def test_set_stats_stores_all_parameters(self): + """Test that set_stats stores all provided parameters.""" + collector = HeatmapCollector(sample_interval_usec=100) + + collector.set_stats( + sample_interval_usec=500, + duration_sec=10.5, + sample_rate=99.5, + error_rate=0.5 + ) + + self.assertEqual(collector.stats['sample_interval_usec'], 500) + self.assertEqual(collector.stats['duration_sec'], 10.5) + self.assertEqual(collector.stats['sample_rate'], 99.5) + self.assertEqual(collector.stats['error_rate'], 0.5) + + def test_set_stats_includes_system_info(self): + """Test that set_stats includes Python and platform info.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + self.assertIn('python_version', collector.stats) + self.assertIn('python_implementation', collector.stats) + self.assertIn('platform', collector.stats) + + def test_set_stats_accepts_kwargs(self): + """Test that set_stats accepts additional kwargs.""" + collector = HeatmapCollector(sample_interval_usec=100) + + collector.set_stats( + sample_interval_usec=100, + duration_sec=1.0, + sample_rate=100.0, + custom_key='custom_value', + another_key=42 + ) + + self.assertEqual(collector.stats['custom_key'], 'custom_value') + self.assertEqual(collector.stats['another_key'], 42) + + def test_set_stats_with_none_error_rate(self): + """Test set_stats with error_rate=None.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + self.assertIn('error_rate', collector.stats) + self.assertIsNone(collector.stats['error_rate']) + + +class TestHeatmapCollectorProcessFrames(unittest.TestCase): + """Test HeatmapCollector.process_frames() method.""" + + def test_process_frames_increments_total_samples(self): + """Test that process_frames increments total samples count.""" + collector = HeatmapCollector(sample_interval_usec=100) + + initial_count = collector._total_samples + frames = [('file.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + self.assertEqual(collector._total_samples, initial_count + 1) + + def test_process_frames_records_line_samples(self): + """Test that process_frames records line samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 5, 'test_func')] + collector.process_frames(frames, thread_id=1) + + # Check that line was recorded + self.assertIn(('test.py', 5), collector.line_samples) + self.assertEqual(collector.line_samples[('test.py', 5)], 1) + + def test_process_frames_records_multiple_lines_in_stack(self): + """Test that process_frames records all lines in a stack.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('file1.py', 10, 'func1'), + ('file2.py', 20, 'func2'), + ('file3.py', 30, 'func3') + ] + collector.process_frames(frames, thread_id=1) + + # All frames should be recorded + self.assertIn(('file1.py', 10), collector.line_samples) + self.assertIn(('file2.py', 20), collector.line_samples) + self.assertIn(('file3.py', 30), collector.line_samples) + + def test_process_frames_distinguishes_self_samples(self): + """Test that process_frames distinguishes self (leaf) samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('leaf.py', 5, 'leaf_func'), # This is the leaf (top of stack) + ('caller.py', 10, 'caller_func') + ] + collector.process_frames(frames, thread_id=1) + + # Leaf should have self sample + self.assertIn(('leaf.py', 5), collector.line_self_samples) + self.assertEqual(collector.line_self_samples[('leaf.py', 5)], 1) + + # Caller should NOT have self sample + self.assertNotIn(('caller.py', 10), collector.line_self_samples) + + def test_process_frames_accumulates_samples(self): + """Test that multiple calls accumulate samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('file.py', 10, 'func')] + + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + + self.assertEqual(collector.line_samples[('file.py', 10)], 3) + self.assertEqual(collector._total_samples, 3) + + def test_process_frames_ignores_invalid_frames(self): + """Test that process_frames ignores invalid frames.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # These should be ignored + invalid_frames = [ + ('', 1, 'test'), + ('[eval]', 1, 'test'), + ('', 1, 'test'), + (None, 1, 'test'), + ('__init__', 0, 'test'), # Special invalid frame + ] + + for frame in invalid_frames: + collector.process_frames([frame], thread_id=1) + + # Should not record these invalid frames + for frame in invalid_frames: + if frame[0]: + self.assertNotIn((frame[0], frame[1]), collector.line_samples) + + def test_process_frames_builds_call_graph(self): + """Test that process_frames builds call graph relationships.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', 5, 'callee_func'), + ('caller.py', 10, 'caller_func') + ] + collector.process_frames(frames, thread_id=1) + + # Check that call relationship was recorded + caller_key = ('caller.py', 10) + self.assertIn(caller_key, collector.call_graph) + + # Check callers graph + callee_key = ('callee.py', 5) + self.assertIn(callee_key, collector.callers_graph) + + def test_process_frames_records_function_definitions(self): + """Test that process_frames records function definition locations.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('module.py', 42, 'my_function')] + collector.process_frames(frames, thread_id=1) + + self.assertIn(('module.py', 'my_function'), collector.function_definitions) + self.assertEqual(collector.function_definitions[('module.py', 'my_function')], 42) + + def test_process_frames_tracks_edge_samples(self): + """Test that process_frames tracks edge sample counts.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', 5, 'callee'), + ('caller.py', 10, 'caller') + ] + + # Process same call stack multiple times + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + + # Check that edge count is tracked + self.assertGreater(len(collector.edge_samples), 0) + + def test_process_frames_handles_empty_frames(self): + """Test that process_frames handles empty frame list.""" + collector = HeatmapCollector(sample_interval_usec=100) + + initial_count = collector._total_samples + collector.process_frames([], thread_id=1) + + # Should still increment total samples + self.assertEqual(collector._total_samples, initial_count + 1) + + def test_process_frames_with_file_samples_dict(self): + """Test that file_samples dict is properly populated.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + self.assertIn('test.py', collector.file_samples) + self.assertIn(10, collector.file_samples['test.py']) + self.assertEqual(collector.file_samples['test.py'][10], 1) + + +class TestHeatmapCollectorExport(unittest.TestCase): + """Test HeatmapCollector.export() method.""" + + def setUp(self): + """Set up test directory.""" + self.test_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.test_dir) + + def test_export_creates_output_directory(self): + """Test that export creates the output directory.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Add some data + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + self.assertTrue(os.path.exists(output_path)) + self.assertTrue(os.path.isdir(output_path)) + + def test_export_creates_index_html(self): + """Test that export creates index.html.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + index_path = os.path.join(output_path, 'index.html') + self.assertTrue(os.path.exists(index_path)) + + def test_export_creates_file_htmls(self): + """Test that export creates individual file HTMLs.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Check for file_XXXX.html files + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + self.assertGreater(len(html_files), 0) + + def test_export_with_empty_data(self): + """Test export with no data collected.""" + collector = HeatmapCollector(sample_interval_usec=100) + + output_path = os.path.join(self.test_dir, 'empty_output') + + # Should handle empty data gracefully + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + def test_export_handles_html_suffix(self): + """Test that export handles .html suffix in output path.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + # Path with .html suffix should be stripped + output_path = os.path.join(self.test_dir, 'output.html') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Should create directory without .html + expected_dir = os.path.join(self.test_dir, 'output') + self.assertTrue(os.path.exists(expected_dir)) + + def test_export_with_multiple_files(self): + """Test export with multiple files.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Add samples for multiple files + collector.process_frames([('file1.py', 10, 'func1')], thread_id=1) + collector.process_frames([('file2.py', 20, 'func2')], thread_id=1) + collector.process_frames([('file3.py', 30, 'func3')], thread_id=1) + + output_path = os.path.join(self.test_dir, 'multi_file') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Should create HTML for each file + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + self.assertGreaterEqual(len(html_files), 3) + + def test_export_index_contains_file_references(self): + """Test that index.html contains references to profiled files.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + frames = [('mytest.py', 10, 'my_func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'test_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + index_path = os.path.join(output_path, 'index.html') + with open(index_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Should contain reference to the file + self.assertIn('mytest', content) + + def test_export_file_html_has_line_numbers(self): + """Test that exported file HTML contains line numbers.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Create a temporary Python file + temp_file = os.path.join(self.test_dir, 'temp_source.py') + with open(temp_file, 'w') as f: + f.write('def test():\n pass\n') + + frames = [(temp_file, 1, 'test')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'line_test') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Find the generated file HTML + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + + if html_files: + with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: + content = f.read() + + # Should have line-related content + self.assertIn('line-', content) + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__(self, thread_id, frame_info): + self.thread_id = thread_id + self.frame_info = frame_info + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" + + +class TestHeatmapCollector(unittest.TestCase): + """Tests for HeatmapCollector functionality.""" + + def test_heatmap_collector_basic(self): + """Test basic HeatmapCollector functionality.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Test empty state + self.assertEqual(len(collector.file_samples), 0) + self.assertEqual(len(collector.line_samples), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo( + 1, + [("file.py", 10, "func1"), ("file.py", 20, "func2")], + )] + ) + ] + collector.collect(test_frames) + + # Should have recorded samples for the file + self.assertGreater(len(collector.line_samples), 0) + self.assertIn("file.py", collector.file_samples) + + # Check that line samples were recorded + file_data = collector.file_samples["file.py"] + self.assertGreater(len(file_data), 0) + + def test_heatmap_collector_export(self): + """Test heatmap HTML export functionality.""" + heatmap_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, heatmap_dir) + + collector = HeatmapCollector(sample_interval_usec=100) + + # Create test data with multiple files + test_frames1 = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])]) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export heatmap + with (captured_stdout(), captured_stderr()): + collector.export(heatmap_dir) + + # Verify index.html was created + index_path = os.path.join(heatmap_dir, "index.html") + self.assertTrue(os.path.exists(index_path)) + self.assertGreater(os.path.getsize(index_path), 0) + + # Check index contains HTML content + with open(index_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("", content.lower()) + self.assertIn(" Date: Wed, 3 Dec 2025 01:16:37 -0600 Subject: [PATCH 384/417] gh-142145: Remove quadratic behavior in node ID cache clearing (GH-142146) * Remove quadratic behavior in node ID cache clearing Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> * Add news fragment --------- Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> --- Lib/test/test_minidom.py | 18 ++++++++++++++++++ Lib/xml/dom/minidom.py | 9 +-------- ...5-12-01-09-36-45.gh-issue-142145.tcAUhg.rst | 1 + 3 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 4f25e9c2a03..4fa5a4e6768 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -2,6 +2,7 @@ import copy import pickle +import time import io from test import support import unittest @@ -173,6 +174,23 @@ def testAppendChild(self): self.assertEqual(dom.documentElement.childNodes[-1].data, "Hello") dom.unlink() + def testAppendChildNoQuadraticComplexity(self): + impl = getDOMImplementation() + + newdoc = impl.createDocument(None, "some_tag", None) + top_element = newdoc.documentElement + children = [newdoc.createElement(f"child-{i}") for i in range(1, 2 ** 15 + 1)] + element = top_element + + start = time.time() + for child in children: + element.appendChild(child) + element = child + end = time.time() + + # This example used to take at least 30 seconds. + self.assertLess(end - start, 1) + def testAppendChildFragment(self): dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() dom.documentElement.appendChild(frag) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index db51f350ea0..0a2ccc00f18 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -292,13 +292,6 @@ def _append_child(self, node): childNodes.append(node) node.parentNode = self -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False def _write_data(writer, text, attr): "Writes datachars to writer." @@ -1555,7 +1548,7 @@ def _clear_id_cache(node): if node.nodeType == Node.DOCUMENT_NODE: node._id_cache.clear() node._id_search_stack = None - elif _in_document(node): + elif node.ownerDocument: node.ownerDocument._id_cache.clear() node.ownerDocument._id_search_stack= None diff --git a/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst new file mode 100644 index 00000000000..440bc7794c6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst @@ -0,0 +1 @@ +Remove quadratic behavior in ``xml.minidom`` node ID cache clearing. From 88cd5d9850d2dc51abe43eb84198904d9870c26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20S=C5=82awecki?= Date: Wed, 3 Dec 2025 10:11:40 +0100 Subject: [PATCH 385/417] gh-142170: Add pymanager link to issue template menu (#142199) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce..de6e8756b03 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" From 4172644d78d58189e46424af0aea302b1d78e2de Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 3 Dec 2025 13:59:14 +0100 Subject: [PATCH 386/417] gh-142206: multiprocessing.resource_tracker: Decode messages using older protocol (GH-142215) --- Lib/multiprocessing/resource_tracker.py | 63 +++++++++++++------ Lib/test/_test_multiprocessing.py | 26 +++++++- ...-12-03-09-36-29.gh-issue-142206.ilwegH.rst | 4 ++ 3 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index b0f9099f4a5..3606d1effb4 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -68,6 +68,13 @@ def __init__(self): self._exitcode = None self._reentrant_messages = deque() + # True to use colon-separated lines, rather than JSON lines, + # for internal communication. (Mainly for testing). + # Filenames not supported by the simple format will always be sent + # using JSON. + # The reader should understand all formats. + self._use_simple_format = False + def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker # gets interrupted by a garbage collection, invoking a finalizer (*) @@ -200,7 +207,9 @@ def _launch(self): os.close(r) def _make_probe_message(self): - """Return a JSON-encoded probe message.""" + """Return a probe message.""" + if self._use_simple_format: + return b'PROBE:0:noop\n' return ( json.dumps( {"cmd": "PROBE", "rtype": "noop"}, @@ -267,6 +276,15 @@ def _write(self, msg): assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" def _send(self, cmd, name, rtype): + if self._use_simple_format and '\n' not in name: + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + self._ensure_running_and_write(msg) + return + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. # POSIX shm_open() and sem_open() require the name, including its leading slash, @@ -286,6 +304,7 @@ def _send(self, cmd, name, rtype): # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" + assert msg.startswith(b'{') self._ensure_running_and_write(msg) @@ -296,6 +315,30 @@ def _send(self, cmd, name, rtype): getfd = _resource_tracker.getfd +def _decode_message(line): + if line.startswith(b'{'): + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + else: + cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1) + name, rtype = rest.rsplit(':', maxsplit=1) + return cmd, rtype, name + + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc @@ -318,23 +361,7 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - try: - obj = json.loads(line.decode('ascii')) - except Exception as e: - raise ValueError("malformed resource_tracker message: %r" % (line,)) from e - - cmd = obj["cmd"] - rtype = obj["rtype"] - b64 = obj.get("base64_name", "") - - if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): - raise ValueError("malformed resource_tracker fields: %r" % (obj,)) - - try: - name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') - except ValueError as e: - raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e - + cmd, rtype, name = _decode_message(line) cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index d718a272318..d03eb1dfb25 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,6 +39,7 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import subTests from test.support.script_helper import assert_python_failure, assert_python_ok # Skip tests if _multiprocessing wasn't built. @@ -4383,6 +4384,19 @@ def test_copy(self): self.assertEqual(bar.z, 2 ** 33) +def resource_tracker_format_subtests(func): + """Run given test using both resource tracker communication formats""" + def _inner(self, *args, **kwargs): + tracker = resource_tracker._resource_tracker + for use_simple_format in False, True: + with ( + self.subTest(use_simple_format=use_simple_format), + unittest.mock.patch.object( + tracker, '_use_simple_format', use_simple_format) + ): + func(self, *args, **kwargs) + return _inner + @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") @hashlib_helper.requires_hashdigest('sha256') class _TestSharedMemory(BaseTestCase): @@ -4662,6 +4676,7 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the # same resource_tracker process as its parent. @@ -4913,6 +4928,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "shared_memory objects to clean up at shutdown", err) @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted @@ -4940,6 +4956,7 @@ def test_shared_memory_untracking(self): mem.close() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_tracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=True, it must cause the memory to be deleted when @@ -7353,13 +7370,18 @@ def test_forkpty(self): @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") class TestSharedMemoryNames(unittest.TestCase): - def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self): + @subTests('use_simple_format', (True, False)) + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors( + self, use_simple_format): # Test script that creates and cleans up shared memory with colon in name test_script = textwrap.dedent(""" import sys from multiprocessing import shared_memory + from multiprocessing import resource_tracker import time + resource_tracker._resource_tracker._use_simple_format = %s + # Test various patterns of colons in names test_names = [ "a:b", @@ -7387,7 +7409,7 @@ def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self sys.exit(1) print("SUCCESS") - """) + """ % use_simple_format) rc, out, err = assert_python_ok("-c", test_script) self.assertIn(b"SUCCESS", out) diff --git a/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst b/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst new file mode 100644 index 00000000000..90e4dd96985 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst @@ -0,0 +1,4 @@ +The resource tracker in the :mod:`multiprocessing` module can now understand +messages from older versions of itself. This avoids issues with upgrading +Python while it is running. (Note that such 'in-place' upgrades are not +tested.) From 7e5fcae09bb0e87ed48cb593f7f46d715e48a102 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Dec 2025 14:33:32 +0100 Subject: [PATCH 387/417] gh-142217: Remove internal _Py_Identifier functions (#142219) Remove internal functions: * _PyDict_ContainsId() * _PyDict_DelItemId() * _PyDict_GetItemIdWithError() * _PyDict_SetItemId() * _PyEval_GetBuiltinId() * _PyObject_CallMethodIdNoArgs() * _PyObject_CallMethodIdObjArgs() * _PyObject_CallMethodIdOneArg() * _PyObject_VectorcallMethodId() * _PyUnicode_EqualToASCIIId() These functions were not exported and so no usable outside CPython. --- Include/internal/pycore_call.h | 33 -------------------- Include/internal/pycore_ceval.h | 2 -- Include/internal/pycore_dict.h | 7 ----- Include/internal/pycore_unicodeobject.h | 8 ----- Objects/call.c | 33 -------------------- Objects/dictobject.c | 41 ------------------------- Objects/odictobject.c | 1 - Objects/unicodeobject.c | 41 ------------------------- Python/ceval.c | 6 ---- 9 files changed, 172 deletions(-) diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 506da06f708..4f4cf02f64b 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -64,39 +64,6 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -extern PyObject* _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} - /* === Vectorcall protocol (PEP 590) ============================= */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 9d81833a234..762d8ef067e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -33,8 +33,6 @@ extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); -extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); - extern void _PyEval_SetSwitchInterval(unsigned long microseconds); extern unsigned long _PyEval_GetSwitchInterval(void); diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index b8fe360321d..1193f496da1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -36,13 +36,6 @@ extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -// "Id" variants -extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); -extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); -extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); -extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); - extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index e7ca65a56b6..97dda73f9b5 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -307,14 +307,6 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( Py_ssize_t seqlen ); -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -extern int _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - // Test whether a unicode is equal to ASCII string. Return 1 if true, // 0 otherwise. The right argument must be ASCII-encoded string. // Any error occurs inside will be cleared before return. diff --git a/Objects/call.c b/Objects/call.c index bd8617825b5..c69015abfb3 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -891,39 +891,6 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) } -PyObject * -_PyObject_CallMethodIdObjArgs(PyObject *obj, _Py_Identifier *name, ...) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (obj == NULL || name == NULL) { - return null_error(tstate); - } - - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return NULL; - } - _PyCStackRef method; - _PyThreadState_PushCStackRef(tstate, &method); - int is_method = _PyObject_GetMethodStackRef(tstate, obj, oname, &method.ref); - if (PyStackRef_IsNull(method.ref)) { - _PyThreadState_PopCStackRef(tstate, &method); - return NULL; - } - PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); - - obj = is_method ? obj : NULL; - - va_list vargs; - va_start(vargs, name); - PyObject *result = object_vacall(tstate, obj, callable, vargs); - va_end(vargs); - - _PyThreadState_PopCStackRef(tstate, &method); - return result; -} - - PyObject * PyObject_CallFunctionObjArgs(PyObject *callable, ...) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 14de21f3c67..ee1c173ae4a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2527,18 +2527,6 @@ _PyDict_GetItemWithError(PyObject *dp, PyObject *kv) return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference } -PyObject * -_PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) -{ - PyObject *kv; - kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return NULL; - Py_hash_t hash = unicode_get_hash(kv); - assert (hash != -1); /* interned strings have their hash value initialised */ - return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference -} - PyObject * _PyDict_GetItemStringWithError(PyObject *v, const char *key) { @@ -4845,16 +4833,6 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return 0; } -int -_PyDict_ContainsId(PyObject *op, _Py_Identifier *key) -{ - PyObject *kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) { - return -1; - } - return PyDict_Contains(op, kv); -} - /* Hack to implement "key in dict" */ static PySequenceMethods dict_as_sequence = { 0, /* sq_length */ @@ -5035,16 +5013,6 @@ PyDict_GetItemStringRef(PyObject *v, const char *key, PyObject **result) return res; } -int -_PyDict_SetItemId(PyObject *v, _Py_Identifier *key, PyObject *item) -{ - PyObject *kv; - kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return -1; - return PyDict_SetItem(v, kv, item); -} - int PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) { @@ -5060,15 +5028,6 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) return err; } -int -_PyDict_DelItemId(PyObject *v, _Py_Identifier *key) -{ - PyObject *kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return -1; - return PyDict_DelItem(v, kv); -} - int PyDict_DelItemString(PyObject *v, const char *key) { diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 45d2ea0203a..25928028919 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -223,7 +223,6 @@ PyDict_DelItem PyMapping_DelItem PyDict_DelItemString PyMapping_DelItemString PyDict_GetItem - PyDict_GetItemWithError PyObject_GetItem -_PyDict_GetItemIdWithError - PyDict_GetItemString PyMapping_GetItemString PyDict_Items PyMapping_Items PyDict_Keys PyMapping_Keys diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7f9f75126a9..f737a885f19 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11194,47 +11194,6 @@ _PyUnicode_EqualToASCIIString(PyObject *unicode, const char *str) memcmp(PyUnicode_1BYTE_DATA(unicode), str, len) == 0; } -int -_PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) -{ - PyObject *right_uni; - - assert(_PyUnicode_CHECK(left)); - assert(right->string); -#ifndef NDEBUG - for (const char *p = right->string; *p; p++) { - assert((unsigned char)*p < 128); - } -#endif - - if (!PyUnicode_IS_ASCII(left)) - return 0; - - right_uni = _PyUnicode_FromId(right); /* borrowed */ - if (right_uni == NULL) { - /* memory error or bad data */ - PyErr_Clear(); - return _PyUnicode_EqualToASCIIString(left, right->string); - } - - if (left == right_uni) - return 1; - - assert(PyUnicode_CHECK_INTERNED(right_uni)); - if (PyUnicode_CHECK_INTERNED(left)) { - return 0; - } - - Py_hash_t right_hash = PyUnicode_HASH(right_uni); - assert(right_hash != -1); - Py_hash_t hash = PyUnicode_HASH(left); - if (hash != -1 && hash != right_hash) { - return 0; - } - - return unicode_eq(left, right_uni); -} - PyObject * PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) { diff --git a/Python/ceval.c b/Python/ceval.c index 5381cd826df..39fb38b7307 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2828,12 +2828,6 @@ _PyEval_GetBuiltin(PyObject *name) return attr; } -PyObject * -_PyEval_GetBuiltinId(_Py_Identifier *name) -{ - return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); -} - PyObject * PyEval_GetLocals(void) { From f6f456f95092142c4b6d038b839975bf7db4d1f2 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 3 Dec 2025 15:24:17 +0100 Subject: [PATCH 388/417] gh-142038: Expand guard for types_world_is_stopped() to fix debug builds without assertions (#142039) --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4c6ff51493f..cbe0215359e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -81,7 +81,7 @@ class object "PyObject *" "&PyBaseObject_Type" #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() -#ifndef NDEBUG +#if !defined(NDEBUG) || defined(Py_DEBUG) // Return true if the world is currently stopped. static bool types_world_is_stopped(void) From aea5531583aaa8bfdf3ebca914e9c694617c3489 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 3 Dec 2025 16:14:53 +0100 Subject: [PATCH 389/417] gh-135676: Reword the f-string (and t-string) section (GH-137469) Much of the information was duplicated in stdtypes.rst; this PR keeps lexical/syntactical details in Lexical Analysis and the evaluation & runtime behaviour in Standard types, with cross-references between the two. Since the t-string section only listed differences from f-strings, and the grammar for the two is equivalent, that section was moved to Standard types almost entirely. Co-authored-by: Blaise Pabon Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/stdtypes.rst | 192 ++++++++------- Doc/reference/expressions.rst | 2 +- Doc/reference/lexical_analysis.rst | 363 +++++++++++++++++------------ 3 files changed, 323 insertions(+), 234 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8b896011734..086da1a705c 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2656,6 +2656,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2664,123 +2666,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: +.. versionadded:: 3.8 - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -.. doctest:: +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' -To include a literal ``{`` or ``}``, use a double bracket: -.. doctest:: +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' - -Functions can also be used, and :ref:`format specifiers `: - -.. doctest:: - - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' - -Any non-string expression is converted using :func:`str`, by default: - -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' -.. doctest:: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" + + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" + + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: +.. _stdtypes-tstrings: -.. doctest:: +Template String Literals (t-strings) +------------------------------------ - >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index c655d6c52ec..165dfa69f88 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -174,7 +174,7 @@ Formally: .. grammar-snippet:: :group: python-grammar - strings: ( `STRING` | fstring)+ | tstring+ + strings: ( `STRING` | `fstring`)+ | `tstring`+ This feature is defined at the syntactical level, so it only works with literals. To concatenate string expressions at run time, the '+' operator may be used:: diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 129dc10d07f..9322d8571f7 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -345,7 +345,15 @@ Whitespace between tokens Except at the beginning of a logical line or in string literals, the whitespace characters space, tab and formfeed can be used interchangeably to separate -tokens. Whitespace is needed between two tokens only if their concatenation +tokens: + +.. grammar-snippet:: + :group: python-grammar + + whitespace: ' ' | tab | formfeed + + +Whitespace is needed between two tokens only if their concatenation could otherwise be interpreted as a different token. For example, ``ab`` is one token, but ``a b`` is two tokens. However, ``+a`` and ``+ a`` both produce two tokens, ``+`` and ``a``, as ``+a`` is not a valid token. @@ -1032,124 +1040,59 @@ f-strings --------- .. versionadded:: 3.6 +.. versionchanged:: 3.7 + The :keyword:`await` and :keyword:`async for` can be used in expressions + within f-strings. +.. versionchanged:: 3.8 + Added the debug specifier (``=``) +.. versionchanged:: 3.12 + Many restrictions on expressions within f-strings have been removed. + Notably, nested strings, comments, and backslashes are now permitted. A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with '``f``' or '``F``'. These strings may contain -replacement fields, which are expressions delimited by curly braces ``{}``. -While other string literals always have a constant value, formatted strings -are really expressions evaluated at run time. +that is prefixed with '``f``' or '``F``'. +Unlike other string literals, f-strings do not have a constant value. +They may contain *replacement fields* delimited by curly braces ``{}``. +Replacement fields contain expressions which are evaluated at run time. +For example:: -Escape sequences are decoded like in ordinary string literals (except when -a literal is also marked as a raw string). After decoding, the grammar -for the contents of the string is: + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' -.. productionlist:: python-grammar - f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" - f_expression: (`conditional_expression` | "*" `or_expr`) - : ("," `conditional_expression` | "," "*" `or_expr`)* [","] - : | `yield_expression` - conversion: "s" | "r" | "a" - format_spec: (`literal_char` | `replacement_field`)* - literal_char: +Any doubled curly braces (``{{`` or ``}}``) outside replacement fields +are replaced with the corresponding single curly brace:: -The parts of the string outside curly braces are treated literally, -except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced -with the corresponding single curly brace. A single opening curly -bracket ``'{'`` marks a replacement field, which starts with a -Python expression. To display both the expression text and its value after -evaluation, (useful in debugging), an equal sign ``'='`` may be added after the -expression. A conversion field, introduced by an exclamation point ``'!'`` may -follow. A format specifier may also be appended, introduced by a colon ``':'``. -A replacement field ends with a closing curly bracket ``'}'``. + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in ordinary +string literals. +This means that escape sequences are decoded (except when a literal is +also marked as a raw string), and newlines are possible in triple-quoted +f-strings:: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\Users\{name}") + C:\Users\Galahad + >>> print(f'''Three shall be the number of the counting + ... and the number of the counting shall be three.''') + Three shall be the number of the counting + and the number of the counting shall be three. Expressions in formatted string literals are treated like regular -Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and both :keyword:`lambda` and -assignment expressions ``:=`` must be surrounded by explicit parentheses. +Python expressions. Each expression is evaluated in the context where the formatted string literal -appears, in order from left to right. Replacement expressions can contain -newlines in both single-quoted and triple-quoted f-strings and they can contain -comments. Everything that comes after a ``#`` inside a replacement field -is a comment (even closing braces and quotes). In that case, replacement fields -must be closed in a different line. - -.. code-block:: text - - >>> f"abc{a # This is a comment }" - ... + 3}" - 'abc5' - -.. versionchanged:: 3.7 - Prior to Python 3.7, an :keyword:`await` expression and comprehensions - containing an :keyword:`async for` clause were illegal in the expressions - in formatted string literals due to a problem with the implementation. - -.. versionchanged:: 3.12 - Prior to Python 3.12, comments were not allowed inside f-string replacement - fields. - -When the equal sign ``'='`` is provided, the output will have the expression -text, the ``'='`` and the evaluated value. Spaces after the opening brace -``'{'``, within the expression and after the ``'='`` are all retained in the -output. By default, the ``'='`` causes the :func:`repr` of the expression to be -provided, unless there is a format specified. When a format is specified it -defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is -declared. - -.. versionadded:: 3.8 - The equal sign ``'='``. - -If a conversion is specified, the result of evaluating the expression -is converted before formatting. Conversion ``'!s'`` calls :func:`str` on -the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. - -The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`~object.__format__` method of the -expression or conversion result. An empty string is passed when the -format specifier is omitted. The formatted result is then included in -the final value of the whole string. - -Top-level format specifiers may include nested replacement fields. These nested -fields may include their own conversion fields and :ref:`format specifiers -`, but may not include more deeply nested replacement fields. The -:ref:`format specifier mini-language ` is the same as that used by -the :meth:`str.format` method. - -Formatted string literals may be concatenated, but replacement fields -cannot be split across literals. - -Some examples of formatted string literals:: - - >>> name = "Fred" - >>> f"He said his name is {name!r}." - "He said his name is 'Fred'." - >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r - "He said his name is 'Fred'." - >>> width = 10 - >>> precision = 4 - >>> value = decimal.Decimal("12.34567") - >>> f"result: {value:{width}.{precision}}" # nested fields - 'result: 12.35' - >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%B %d, %Y}" # using date format specifier - 'January 27, 2017' - >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging - 'today=January 27, 2017' - >>> number = 1024 - >>> f"{number:#0x}" # using integer format specifier - '0x400' - >>> foo = "bar" - >>> f"{ foo = }" # preserves whitespace - " foo = 'bar'" - >>> line = "The mill's closed" - >>> f"{line = }" - 'line = "The mill\'s closed"' - >>> f"{line = :20}" - "line = The mill's closed " - >>> f"{line = !r:20}" - 'line = "The mill\'s closed" ' +appears, in order from left to right. +An empty expression is not allowed, and both :keyword:`lambda` and +assignment expressions ``:=`` must be surrounded by explicit parentheses:: + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' Reusing the outer f-string quoting type inside a replacement field is permitted:: @@ -1158,10 +1101,6 @@ permitted:: >>> f"abc {a["x"]} def" 'abc 2 def' -.. versionchanged:: 3.12 - Prior to Python 3.12, reuse of the same quoting type of the outer f-string - inside a replacement field was not possible. - Backslashes are also allowed in replacement fields and are evaluated the same way as in any other context:: @@ -1172,23 +1111,84 @@ way as in any other context:: b c -.. versionchanged:: 3.12 - Prior to Python 3.12, backslashes were not permitted inside an f-string - replacement field. +It is possible to nest f-strings:: -Formatted string literals cannot be used as docstrings, even if they do not -include expressions. + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' -:: +Portable Python programs should not use more than 5 levels of nesting. + +.. impl-detail:: + + CPython does not limit nesting of f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. +Everything that comes after a ``#`` inside a replacement field +is a comment (even closing braces and quotes). +This means that replacement fields with comments must be closed in a +different line: + +.. code-block:: text + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* -- an equal sign (``=``), optionally surrounded by + whitespace on one or both sides; +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). + +See the :ref:`Standard Library section on f-strings ` +for details on how these fields are evaluated. + +As that section explains, *format specifiers* are passed as the second argument +to the :func:`format` function to format a replacement field value. +For example, they can be used to specify a field width and padding characters +using the :ref:`Format Specification Mini-Language `:: + + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' + +Top-level format specifiers may include nested replacement fields:: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and +:ref:`format specifiers `:: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested replacement +fields. + +Formatted string literals cannot be used as :term:`docstrings `, +even if they do not include expressions:: >>> def foo(): ... f"Not a docstring" ... - >>> foo.__doc__ is None - True + >>> print(foo.__doc__) + None -See also :pep:`498` for the proposal that added formatted string literals, -and :meth:`str.format`, which uses a related format string mechanism. +.. seealso:: + + * :pep:`498` -- Literal String Interpolation + * :pep:`701` -- Syntactic formalization of f-strings + * :meth:`str.format`, which uses a related format string mechanism. .. _t-strings: @@ -1201,36 +1201,99 @@ t-strings A :dfn:`template string literal` or :dfn:`t-string` is a string literal that is prefixed with '``t``' or '``T``'. -These strings follow the same syntax and evaluation rules as -:ref:`formatted string literals `, with the following differences: +These strings follow the same syntax rules as +:ref:`formatted string literals `. +For differences in evaluation rules, see the +:ref:`Standard Library section on t-strings ` -* Rather than evaluating to a ``str`` object, template string literals evaluate - to a :class:`string.templatelib.Template` object. -* The :func:`format` protocol is not used. - Instead, the format specifier and conversions (if any) are passed to - a new :class:`~string.templatelib.Interpolation` object that is created - for each evaluated expression. - It is up to code that processes the resulting :class:`~string.templatelib.Template` - object to decide how to handle format specifiers and conversions. +Formal grammar for f-strings +---------------------------- -* Format specifiers containing nested replacement fields are evaluated eagerly, - prior to being passed to the :class:`~string.templatelib.Interpolation` object. - For instance, an interpolation of the form ``{amount:.{precision}f}`` will - evaluate the inner expression ``{precision}`` to determine the value of the - ``format_spec`` attribute. - If ``precision`` were to be ``2``, the resulting format specifier - would be ``'.2f'``. +F-strings are handled partly by the :term:`lexical analyzer`, which produces the +tokens :py:data:`~token.FSTRING_START`, :py:data:`~token.FSTRING_MIDDLE` +and :py:data:`~token.FSTRING_END`, and partly by the parser, which handles +expressions in the replacement field. +The exact way the work is split is a CPython implementation detail. -* When the equals sign ``'='`` is provided in an interpolation expression, - the text of the expression is appended to the literal string that precedes - the relevant interpolation. - This includes the equals sign and any surrounding whitespace. - The :class:`!Interpolation` instance for the expression will be created as - normal, except that :attr:`~string.templatelib.Interpolation.conversion` will - be set to '``r``' (:func:`repr`) by default. - If an explicit conversion or format specifier are provided, - this will override the default behaviour. +Correspondingly, the f-string grammar is a mix of +:ref:`lexical and syntactic definitions `. + +Whitespace is significant in these situations: + +* There may be no whitespace in :py:data:`~token.FSTRING_START` (between + the prefix and quote). +* Whitespace in :py:data:`~token.FSTRING_MIDDLE` is part of the literal + string contents. +* In ``fstring_replacement_field``, if ``f_debug_specifier`` is present, + all whitespace after the opening brace until the ``f_debug_specifier``, + as well as whitespace immediatelly following ``f_debug_specifier``, + is retained as part of the expression. + + .. impl-detail:: + + The expression is not handled in the tokenization phase; it is + retrieved from the source code using locations of the ``{`` token + and the token after ``=``. + + +The ``FSTRING_MIDDLE`` definition uses +:ref:`negative lookaheads ` (``!``) +to indicate special characters (backslash, newline, ``{``, ``}``) and +sequences (``f_quote``). + +.. grammar-snippet:: + :group: python-grammar + + fstring: `FSTRING_START` `fstring_middle`* `FSTRING_END` + + FSTRING_START: `fstringprefix` ("'" | '"' | "'''" | '"""') + FSTRING_END: `f_quote` + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | `fstring_replacement_field` + | `FSTRING_MIDDLE` + FSTRING_MIDDLE: + | (!"\" !`newline` !'{' !'}' !`f_quote`) `source_character` + | `stringescapeseq` + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' `f_expression` [`f_debug_specifier`] [`fstring_conversion`] + [`fstring_full_format_spec`] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' `fstring_format_spec`* + fstring_format_spec: + | `FSTRING_MIDDLE` + | `fstring_replacement_field` + f_expression: + | ','.(`conditional_expression` | "*" `or_expr`)+ [","] + | `yield_expression` + +.. note:: + + In the above grammar snippet, the ``f_quote`` and ``FSTRING_MIDDLE`` rules + are context-sensitive -- they depend on the contents of ``FSTRING_START`` + of the nearest enclosing ``fstring``. + + Constructing a more traditional formal grammar from this template is left + as an exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with *t* +instead of *f* at the beginning of rule and token names and in the prefix. + +.. grammar-snippet:: + :group: python-grammar + + tstring: TSTRING_START tstring_middle* TSTRING_END + + .. _numbers: From 62423c9c36f428ba07c83aeea7cbacc7cbb34ed2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 3 Dec 2025 17:43:35 +0000 Subject: [PATCH 390/417] GH-141794: Limit size of generated machine code. (GH-142228) * Factor out bodies of the largest uops, to reduce jit code size. * Factor out common assert, also reducing jit code size. * Limit size of jitted code for a single executor to 1MB. --- Include/internal/pycore_ceval.h | 58 ++ Include/internal/pycore_jit.h | 3 + Include/internal/pycore_uop.h | 5 + Lib/test/test_generated_cases.py | 54 +- Python/bytecodes.c | 179 ++--- Python/ceval.c | 284 +++++++- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 1094 +++++++++++------------------- Python/generated_cases.c.h | 1070 ++++++++++------------------- Python/jit.c | 4 + Python/optimizer_analysis.c | 30 + Python/optimizer_cases.c.h | 346 +++++----- Tools/cases_generator/stack.py | 2 +- Tools/jit/template.c | 6 + 14 files changed, 1406 insertions(+), 1731 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 762d8ef067e..6bf33bddd5b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -408,6 +408,64 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; +/* Helper functions for large uops */ + +PyAPI_FUNC(PyObject *) +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFast_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFastWithKeywords_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFast_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_CallBuiltinClass_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args); + +PyAPI_FUNC(void) +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607b..b1550a6ddcf 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,6 +13,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +/* To be able to reason about code layout and branches, keep code size below 1 MB */ +#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1) + #ifdef _Py_JIT typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4e1b15af42c..70576046385 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -36,7 +36,12 @@ typedef struct _PyUOpInstruction{ } _PyUOpInstruction; // This is the length of the trace we translate initially. +#ifdef Py_DEBUG + // With asserts, the stencils are a lot larger +#define UOP_MAX_TRACE_LENGTH 1000 +#else #define UOP_MAX_TRACE_LENGTH 3000 +#endif #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) /* Bloom filter with m = 256 diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 09ce329bdcd..ac62e11c274 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -165,7 +165,7 @@ def test_inst_one_pop(self): value = stack_pointer[-1]; SPAM(value); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -190,7 +190,7 @@ def test_inst_one_push(self): res = SPAM(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -247,7 +247,7 @@ def test_binary_op(self): res = SPAM(left, right); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -366,14 +366,14 @@ def test_sync_sp(self): _PyStackRef res; arg = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); escaping_call(); stack_pointer = _PyFrame_GetStackPointer(frame); res = Py_None; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -489,7 +489,7 @@ def test_error_if_pop(self): res = 0; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -523,7 +523,7 @@ def test_error_if_pop_with_result(self): } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -553,7 +553,7 @@ def test_cache_effect(self): uint32_t extra = read_u32(&this_instr[2].cache); (void)extra; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -640,7 +640,7 @@ def test_macro_instruction(self): } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -688,7 +688,7 @@ def test_macro_instruction(self): stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -827,7 +827,7 @@ def test_array_input(self): below = stack_pointer[-2 - oparg*2]; SPAM(values, oparg); stack_pointer += -2 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -860,7 +860,7 @@ def test_array_output(self): stack_pointer[-2] = below; stack_pointer[-1 + oparg*3] = above; stack_pointer += oparg*3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -889,7 +889,7 @@ def test_array_input_output(self): above = 0; stack_pointer[0] = above; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -918,11 +918,11 @@ def test_array_error_if(self): extra = stack_pointer[-1 - oparg]; if (oparg == 0) { stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -960,7 +960,7 @@ def test_macro_push_push(self): stack_pointer[0] = val1; stack_pointer[1] = val2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1263,13 +1263,13 @@ def test_flush(self): stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); // SECOND { USE(a, b); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1325,7 +1325,7 @@ def test_pop_on_error_peeks(self): } } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1368,14 +1368,14 @@ def test_push_then_error(self): stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1661,7 +1661,7 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): stack_pointer[0] = out1; stack_pointer[1] = out2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1881,7 +1881,7 @@ def test_reassigning_dead_inputs(self): stack_pointer = _PyFrame_GetStackPointer(frame); in = temp; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(in); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2116,7 +2116,7 @@ def test_validate_uop_unused_input(self): output = """ case OP: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ @@ -2133,7 +2133,7 @@ def test_validate_uop_unused_input(self): output = """ case OP: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ @@ -2155,7 +2155,7 @@ def test_validate_uop_unused_output(self): foo = NULL; stack_pointer[0] = foo; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ @@ -2173,7 +2173,7 @@ def test_validate_uop_unused_output(self): output = """ case OP: { stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 01cd1e83598..4ba255d28bd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1976,14 +1976,8 @@ dummy_func( } inst(BUILD_STRING, (pieces[oparg] -- str)) { - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - DECREF_INPUTS(); + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); + DEAD(pieces); ERROR_IF(str_o == NULL); str = PyStackRef_FromPyObjectSteal(str_o); } @@ -2098,17 +2092,9 @@ dummy_func( } inst(BUILD_MAP, (values[oparg*2] -- map)) { - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - DECREF_INPUTS(); + + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); + DEAD(values); ERROR_IF(map_o == NULL); map = PyStackRef_FromPyObjectStealMortal(map_o); } @@ -3891,27 +3877,20 @@ dummy_func( #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4186,14 +4165,13 @@ dummy_func( } DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - DECREF_INPUTS(); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4241,31 +4219,24 @@ dummy_func( op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4278,30 +4249,20 @@ dummy_func( op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); - /* res = func(self, arguments, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4468,19 +4429,16 @@ dummy_func( assert(self != NULL); EXIT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); - int nargs = total_args - 1; - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4548,18 +4506,16 @@ dummy_func( assert(self != NULL); EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); - int nargs = total_args - 1; - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4792,30 +4748,21 @@ dummy_func( #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); + DEAD(kwnames); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } diff --git a/Python/ceval.c b/Python/ceval.c index 39fb38b7307..1709dda0cbe 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,6 +1017,281 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #include "ceval_macros.h" + +/* Helper functions to keep the size of the largest uops down */ + +PyObject * +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args; + if (kwnames_o != NULL) { + positional_args -= (int)PyTuple_GET_SIZE(kwnames_o); + } + res = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + PyStackRef_XCLOSE(kwnames); + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuiltinCallFast_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + res = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args + ); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuiltinCallFastWithKeywords_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + res = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_PyCallMethodDescriptorFast_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); + assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); + + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + res = cfunc(self, (args_o + 1), total_args - 1); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); + assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); + + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + res = cfunc(self, (args_o + 1), total_args-1, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_CallBuiltinClass_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(callable); + res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + res = _PyUnicode_JoinArray(&_Py_STR(empty), args_o, total_args); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + return res; +} + + + +PyObject * +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, half_args*2, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + res = _PyDict_FromItems( + args_o, 2, + args_o+1, 2, + half_args + ); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = half_args*2-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + return res; +} + +#ifdef Py_DEBUG +void +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno +) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + return; + } + int level = (int)(stack_pointer - _PyFrame_Stackbase(frame)); + if (level < 0) { + printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } + int size = _PyFrame_GetCode(frame)->co_stacksize; + if (level > size) { + printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } +} +#endif + int _Py_CheckRecursiveCallPy( PyThreadState *tstate) { @@ -1078,11 +1353,12 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject if (result == NULL) { return NULL; } - result++; } else { result = scratch; } + result++; + result[0] = NULL; /* Keep GCC happy */ for (int i = 0; i < nargs; i++) { result[i] = PyStackRef_AsPyObjectBorrow(input[i]); } @@ -1097,6 +1373,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + #if _Py_TIER2 // 0 for success, -1 for error. static int diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c30638c221a..edf8fc9a57d 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -458,7 +458,7 @@ do { \ #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ - PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp); #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e1edd20b778..7273a87681b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -84,7 +84,7 @@ value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -96,7 +96,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -108,7 +108,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -120,7 +120,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -132,7 +132,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -144,7 +144,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -156,7 +156,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -168,7 +168,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -180,7 +180,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -191,7 +191,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -203,7 +203,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -215,7 +215,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -227,7 +227,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -239,7 +239,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -251,7 +251,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -263,7 +263,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -275,7 +275,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -287,7 +287,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -298,7 +298,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -309,7 +309,7 @@ GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -320,7 +320,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -333,7 +333,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -346,7 +346,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -359,7 +359,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -372,7 +372,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -384,7 +384,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -396,7 +396,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -411,7 +411,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -426,7 +426,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -441,7 +441,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -456,7 +456,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -471,7 +471,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -486,7 +486,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -501,7 +501,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -515,7 +515,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -526,7 +526,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -539,7 +539,7 @@ assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -549,7 +549,7 @@ assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -559,7 +559,7 @@ assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -569,7 +569,7 @@ assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -579,12 +579,12 @@ tos = stack_pointer[-1]; nos = stack_pointer[-2]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(tos); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(nos); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -596,7 +596,7 @@ res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -604,7 +604,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -618,7 +618,7 @@ iter = stack_pointer[-2]; (void)index_or_null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -634,7 +634,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -649,7 +649,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -659,7 +659,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -682,7 +682,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -692,7 +692,7 @@ res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -723,7 +723,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -833,7 +833,7 @@ else { assert(Py_SIZE(value_o)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -849,14 +849,14 @@ _PyStackRef res; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -868,7 +868,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -878,7 +878,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -949,7 +949,7 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -974,7 +974,7 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -999,7 +999,7 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1043,12 +1043,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1070,12 +1070,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1097,12 +1097,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1124,12 +1124,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1151,12 +1151,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1178,12 +1178,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1203,13 +1203,13 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1240,7 +1240,7 @@ PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1304,11 +1304,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1330,7 +1330,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); @@ -1338,7 +1338,7 @@ stack_pointer += 2; } stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1348,7 +1348,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1371,7 +1371,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); @@ -1389,7 +1389,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1443,7 +1443,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1472,14 +1472,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1511,14 +1511,14 @@ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectBorrow(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1569,7 +1569,7 @@ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; tuple_st = res; @@ -1631,14 +1631,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1673,7 +1673,7 @@ STAT_INC(BINARY_OP, hit); stack_pointer[0] = getitem; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1692,7 +1692,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-3] = new_frame; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1706,11 +1706,11 @@ PyStackRef_AsPyObjectSteal(v)); if (err < 0) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1726,11 +1726,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1757,7 +1757,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1799,7 +1799,7 @@ UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); @@ -1823,7 +1823,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1851,7 +1851,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1868,7 +1868,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1878,7 +1878,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1904,14 +1904,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1922,7 +1922,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -1935,7 +1935,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1958,7 +1958,7 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1968,7 +1968,7 @@ iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1989,7 +1989,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2006,7 +2006,7 @@ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2019,7 +2019,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2029,7 +2029,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2078,7 +2078,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -2101,7 +2101,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2115,7 +2115,7 @@ ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2126,7 +2126,7 @@ value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2149,7 +2149,7 @@ bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2166,7 +2166,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2183,7 +2183,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2227,7 +2227,7 @@ top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -2236,7 +2236,7 @@ JUMP_TO_ERROR(); } stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2259,7 +2259,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2284,7 +2284,7 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2317,7 +2317,7 @@ } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2332,7 +2332,7 @@ top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); @@ -2341,7 +2341,7 @@ JUMP_TO_ERROR(); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2365,7 +2365,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -2381,7 +2381,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2400,7 +2400,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2442,7 +2442,7 @@ locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2461,7 +2461,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2477,7 +2477,7 @@ JUMP_TO_ERROR(); } stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2489,7 +2489,7 @@ null[0] = PyStackRef_NULL; } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2543,7 +2543,7 @@ STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2580,7 +2580,7 @@ STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2663,14 +2663,14 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2684,7 +2684,7 @@ if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2692,7 +2692,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2705,7 +2705,7 @@ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2729,39 +2729,18 @@ _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2789,7 +2768,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(format[0]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2798,12 +2777,12 @@ stack_pointer += -(oparg & 1); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2813,7 +2792,7 @@ interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2829,12 +2808,12 @@ PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(interpolations); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2844,7 +2823,7 @@ template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2860,7 +2839,7 @@ tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2878,7 +2857,7 @@ list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2908,7 +2887,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2916,7 +2895,7 @@ } assert(Py_IsNone(none_val)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2934,7 +2913,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2962,7 +2941,7 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } int err = 0; @@ -2982,7 +2961,7 @@ } if (err) { stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2991,7 +2970,7 @@ set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3000,44 +2979,18 @@ _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3103,14 +3056,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3136,14 +3089,14 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3169,11 +3122,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3216,14 +3169,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_ERROR(); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3264,7 +3217,7 @@ self_or_null = self_st; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3272,7 +3225,7 @@ stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; @@ -3284,12 +3237,12 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3313,7 +3266,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3329,7 +3282,7 @@ PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3340,7 +3293,7 @@ stack_pointer += 1; } stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3672,7 +3625,7 @@ } UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3731,7 +3684,7 @@ UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3756,7 +3709,7 @@ FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3786,7 +3739,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } @@ -3805,7 +3758,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3827,7 +3780,7 @@ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3853,7 +3806,7 @@ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3877,7 +3830,7 @@ res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3900,11 +3853,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3929,14 +3882,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3974,14 +3927,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4008,14 +3961,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4043,7 +3996,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } PyObject *match_o = NULL; @@ -4061,7 +4014,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } @@ -4079,7 +4032,7 @@ stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4102,14 +4055,14 @@ int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4135,14 +4088,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4161,7 +4114,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4208,7 +4161,7 @@ len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4241,7 +4194,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -4254,7 +4207,7 @@ } stack_pointer[0] = attrs; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4266,7 +4219,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4278,7 +4231,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4298,7 +4251,7 @@ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4323,7 +4276,7 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4337,7 +4290,7 @@ stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4403,7 +4356,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4476,7 +4429,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4525,7 +4478,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4580,7 +4533,7 @@ next = PyStackRef_FromPyObjectSteal(res); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4616,7 +4569,7 @@ gen_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[0] = gen_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4628,7 +4581,7 @@ method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4693,7 +4646,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4715,7 +4668,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4762,7 +4715,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4783,7 +4736,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4797,14 +4750,14 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4819,14 +4772,14 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4859,7 +4812,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4916,14 +4869,14 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); stack_pointer[0] = new_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5034,68 +4987,28 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5204,7 +5117,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5229,7 +5142,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5254,7 +5167,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5279,7 +5192,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5304,7 +5217,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5328,7 +5241,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5338,7 +5251,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -5409,7 +5322,7 @@ res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5445,7 +5358,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5455,7 +5368,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5488,7 +5401,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5498,7 +5411,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5572,7 +5485,7 @@ tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); @@ -5584,7 +5497,7 @@ init_frame = PyStackRef_Wrap(temp); stack_pointer[0] = init_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5600,7 +5513,7 @@ JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5630,57 +5543,21 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5727,7 +5604,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5737,7 +5614,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5750,13 +5627,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5766,62 +5643,22 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5834,13 +5671,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5851,61 +5688,17 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5944,19 +5737,19 @@ JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5993,17 +5786,17 @@ } (void)null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(cls); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(instance); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6011,7 +5804,7 @@ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6045,12 +5838,12 @@ int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6132,14 +5925,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6181,61 +5974,24 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6290,7 +6046,7 @@ PyStackRef_CLOSE(self_stackref); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6300,7 +6056,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6341,60 +6097,24 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6456,19 +6176,19 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); stack_pointer[0] = new_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6569,78 +6289,28 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6686,7 +6356,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6698,7 +6368,7 @@ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[0] = func; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6720,7 +6390,7 @@ *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6752,7 +6422,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6774,14 +6444,14 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6797,7 +6467,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6807,7 +6477,7 @@ result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6821,7 +6491,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6836,7 +6506,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6858,14 +6528,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6876,7 +6546,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6887,7 +6557,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6898,7 +6568,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6910,7 +6580,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6942,7 +6612,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7009,7 +6679,7 @@ flag = stack_pointer[-1]; int is_true = PyStackRef_IsTrue(flag); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!is_true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -7022,7 +6692,7 @@ flag = stack_pointer[-1]; int is_false = PyStackRef_IsFalse(flag); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!is_false) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -7036,7 +6706,7 @@ int is_none = PyStackRef_IsNone(val); if (!is_none) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7046,7 +6716,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7055,7 +6725,7 @@ val = stack_pointer[-1]; int is_none = PyStackRef_IsNone(val); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7162,7 +6832,7 @@ value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7172,14 +6842,14 @@ pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7189,7 +6859,7 @@ value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7200,7 +6870,7 @@ callable = stack_pointer[-2]; (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7215,13 +6885,13 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7238,18 +6908,18 @@ null = stack_pointer[-3]; callable = stack_pointer[-4]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7262,14 +6932,14 @@ pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7281,19 +6951,19 @@ pop1 = stack_pointer[-2]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7306,14 +6976,14 @@ PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7327,20 +6997,20 @@ callable = stack_pointer[-3]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7356,25 +7026,25 @@ callable = stack_pointer[-4]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7389,7 +7059,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7404,7 +7074,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 57d5e71144d..68d73cccec4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -76,7 +76,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } DISPATCH(); } @@ -135,7 +135,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -195,7 +195,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -255,7 +255,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -314,12 +314,12 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -386,7 +386,7 @@ PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -460,7 +460,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -520,7 +520,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -578,7 +578,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_LABEL(error); } @@ -586,7 +586,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -661,7 +661,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -759,7 +759,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } DISPATCH(); } @@ -824,7 +824,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -832,7 +832,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -902,7 +902,7 @@ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -910,7 +910,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -976,7 +976,7 @@ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; tuple_st = res; @@ -1041,7 +1041,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1101,7 +1101,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1138,7 +1138,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); @@ -1146,7 +1146,7 @@ stack_pointer += 2; } stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1157,7 +1157,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1191,7 +1191,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(format[0]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1200,12 +1200,12 @@ stack_pointer += -(oparg & 1); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1215,7 +1215,7 @@ interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1239,7 +1239,7 @@ list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1254,44 +1254,18 @@ _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1319,7 +1293,7 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } int err = 0; @@ -1339,7 +1313,7 @@ } if (err) { stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1348,7 +1322,7 @@ set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1376,14 +1350,14 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1398,39 +1372,18 @@ _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1453,12 +1406,12 @@ PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(interpolations); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1468,7 +1421,7 @@ template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1490,7 +1443,7 @@ tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1583,7 +1536,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -1611,7 +1564,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer[-2 - oparg] = callable; @@ -1664,7 +1617,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -1674,7 +1627,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1778,7 +1731,7 @@ tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); @@ -1936,7 +1889,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -2046,7 +1999,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -2118,60 +2071,24 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2204,13 +2121,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2222,65 +2139,25 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2313,13 +2190,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2332,64 +2209,20 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2460,7 +2293,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2473,7 +2306,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2592,14 +2425,14 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -2617,17 +2450,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2640,7 +2473,7 @@ { stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2667,7 +2500,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2677,7 +2510,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2709,14 +2542,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2773,17 +2606,17 @@ } (void)null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(cls); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(instance); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2792,7 +2625,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2876,7 +2709,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2911,7 +2744,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer[-3 - oparg] = callable; @@ -2966,7 +2799,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -2974,7 +2807,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3072,12 +2905,12 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -3150,81 +2983,31 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3309,12 +3092,12 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -3400,12 +3183,12 @@ JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3413,7 +3196,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3479,12 +3262,12 @@ int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3554,63 +3337,27 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3676,64 +3423,27 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3813,7 +3523,7 @@ PyStackRef_CLOSE(self_stackref); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3826,7 +3536,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3922,7 +3632,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3932,7 +3642,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3983,71 +3693,31 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4158,7 +3828,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -4240,7 +3910,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -4319,7 +3989,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4332,7 +4002,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4391,7 +4061,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4404,7 +4074,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4462,7 +4132,7 @@ res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4501,7 +4171,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; @@ -4519,7 +4189,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4537,7 +4207,7 @@ stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4567,14 +4237,14 @@ int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4621,7 +4291,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); none = PyStackRef_None; } else { @@ -4633,7 +4303,7 @@ stack_pointer[0] = none; stack_pointer[1] = value; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4686,7 +4356,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4706,7 +4376,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4761,7 +4431,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4820,7 +4490,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4879,7 +4549,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4931,7 +4601,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4939,7 +4609,7 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4989,7 +4659,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4997,7 +4667,7 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5047,7 +4717,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5055,7 +4725,7 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5077,7 +4747,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5087,7 +4757,7 @@ result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5105,7 +4775,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5145,7 +4815,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5285,7 +4955,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -5317,14 +4987,14 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5360,14 +5030,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5407,7 +5077,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); @@ -5429,7 +5099,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5452,7 +5122,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5518,7 +5188,7 @@ JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5555,7 +5225,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5570,7 +5240,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5599,14 +5269,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5660,7 +5330,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5815,7 +5485,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5882,7 +5552,7 @@ } stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5938,7 +5608,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5968,7 +5638,7 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5978,7 +5648,7 @@ iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5999,7 +5669,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6023,7 +5693,7 @@ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6042,7 +5712,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6052,7 +5722,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6084,7 +5754,7 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6098,7 +5768,7 @@ stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6126,7 +5796,7 @@ len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6196,7 +5866,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6228,14 +5898,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6329,7 +5999,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -6355,7 +6025,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6406,7 +6076,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6416,7 +6086,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6535,14 +6205,14 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -6560,17 +6230,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6583,7 +6253,7 @@ { stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6682,7 +6352,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6715,7 +6385,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6768,7 +6438,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6776,7 +6446,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6820,7 +6490,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); @@ -6855,7 +6525,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6889,7 +6559,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6928,7 +6598,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7088,7 +6758,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } @@ -7133,7 +6803,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } @@ -7156,7 +6826,7 @@ } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7194,7 +6864,7 @@ (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7221,7 +6891,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7245,14 +6915,14 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7273,7 +6943,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7305,7 +6975,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7420,7 +7090,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -7434,7 +7104,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7478,7 +7148,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -7502,7 +7172,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7530,7 +7200,7 @@ if (!PyStackRef_IsNull(executor)) { tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(executor); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7566,11 +7236,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7738,7 +7408,7 @@ JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7774,7 +7444,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7782,7 +7452,7 @@ } assert(Py_IsNone(none_val)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7838,7 +7508,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7854,7 +7524,7 @@ PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7866,7 +7536,7 @@ } } stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7924,7 +7594,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7992,7 +7662,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8048,7 +7718,7 @@ tstate, PyStackRef_FromPyObjectNew(f), 2, frame); new_frame->localsplus[0] = owner; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = 10u ; DISPATCH_INLINED(new_frame); @@ -8130,7 +7800,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8186,7 +7856,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8232,7 +7902,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8299,7 +7969,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8373,7 +8043,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8412,7 +8082,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8420,7 +8090,7 @@ } stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8480,7 +8150,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8488,7 +8158,7 @@ } stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8573,7 +8243,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -8651,7 +8321,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8761,7 +8431,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8791,7 +8461,7 @@ bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8808,7 +8478,7 @@ value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8825,7 +8495,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8845,7 +8515,7 @@ if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8853,7 +8523,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8870,7 +8540,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8887,7 +8557,7 @@ GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8904,7 +8574,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8925,7 +8595,7 @@ stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8951,7 +8621,7 @@ value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8972,7 +8642,7 @@ stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9010,14 +8680,14 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9038,7 +8708,7 @@ int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(mod_or_class_dict); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9092,7 +8762,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9148,7 +8818,7 @@ } } stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9228,7 +8898,7 @@ } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9295,7 +8965,7 @@ } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9319,7 +8989,7 @@ locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9342,7 +9012,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9360,7 +9030,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9386,7 +9056,7 @@ method_and_self = &stack_pointer[-1]; PyObject *name = _Py_SpecialMethods[oparg].name; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9475,7 +9145,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } @@ -9520,7 +9190,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } @@ -9543,7 +9213,7 @@ } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9598,14 +9268,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9658,7 +9328,7 @@ self_or_null = self_st; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9666,7 +9336,7 @@ stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; @@ -9678,12 +9348,12 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9725,7 +9395,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9737,7 +9407,7 @@ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[0] = func; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9768,7 +9438,7 @@ JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9807,7 +9477,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -9820,7 +9490,7 @@ } stack_pointer[0] = attrs; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9847,7 +9517,7 @@ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9866,7 +9536,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9885,7 +9555,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9928,7 +9598,7 @@ ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9946,7 +9616,7 @@ iter = stack_pointer[-2]; (void)index_or_null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9971,7 +9641,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10014,7 +9684,7 @@ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10057,7 +9727,7 @@ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10079,7 +9749,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10094,7 +9764,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10126,7 +9796,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10142,7 +9812,7 @@ res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10162,7 +9832,7 @@ PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = do_raise(tstate, exc, cause); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10196,7 +9866,7 @@ } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); @@ -10365,7 +10035,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10383,7 +10053,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -10396,7 +10066,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10445,7 +10115,7 @@ _PyInterpreterFrame *gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -10485,7 +10155,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10493,7 +10163,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10501,7 +10171,7 @@ } stack_pointer[0] = retval; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10561,7 +10231,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -10640,7 +10310,7 @@ JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10668,7 +10338,7 @@ *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10689,7 +10359,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10748,7 +10418,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -10823,7 +10493,7 @@ } UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -10874,7 +10544,7 @@ FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -10962,7 +10632,7 @@ UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -10986,7 +10656,7 @@ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11003,7 +10673,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11050,14 +10720,14 @@ _PyStackRef tmp = GETLOCAL(oparg1); GETLOCAL(oparg1) = value1; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11079,7 +10749,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11108,7 +10778,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11125,7 +10795,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11169,7 +10839,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); @@ -11187,7 +10857,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -11246,7 +10916,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -11294,7 +10964,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11377,7 +11047,7 @@ UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); @@ -11443,7 +11113,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11454,7 +11124,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11489,7 +11159,7 @@ { value = owner; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11497,7 +11167,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11554,7 +11224,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11671,7 +11341,7 @@ else { assert(Py_SIZE(value_o)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11760,7 +11430,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11770,7 +11440,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11789,7 +11459,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11799,7 +11469,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11835,7 +11505,7 @@ top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); @@ -11844,7 +11514,7 @@ JUMP_TO_LABEL(error); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11885,7 +11555,7 @@ top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -11895,7 +11565,7 @@ } } stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11950,7 +11620,7 @@ } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12000,7 +11670,7 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12051,7 +11721,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12100,7 +11770,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -12123,7 +11793,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -12146,7 +11816,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/jit.c b/Python/jit.c index 7ab0f8ddd43..47d3d7a5d27 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -60,6 +60,10 @@ jit_error(const char *message) static unsigned char * jit_alloc(size_t size) { + if (size > PY_MAX_JIT_CODE_SIZE) { + jit_error("code too big; refactor bytecodes.c to keep uop size down, or reduce maximum trace length."); + return NULL; + } assert(size); assert(size % get_page_size() == 0); #ifdef MS_WINDOWS diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 8d7b734e17c..685659ef7c4 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -278,6 +278,36 @@ get_co_name(JitOptContext *ctx, int index) return PyTuple_GET_ITEM(get_current_code_object(ctx)->co_names, index); } +#ifdef Py_DEBUG +void +_Py_opt_assert_within_stack_bounds( + _Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer, + const char *filename, int lineno +) { + if (frame->code == ((PyCodeObject *)&_Py_InitCleanup)) { + return; + } + int level = (int)(stack_pointer - frame->stack); + if (level < 0) { + printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } + int size = (int)(frame->stack_len); + if (level > size) { + printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } +} +#endif + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_opt_assert_within_stack_bounds(ctx->frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + // TODO (gh-134584) generate most of this table automatically const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9ebd113df2d..0c2d34d2e64 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -35,7 +35,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -44,7 +44,7 @@ value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -53,7 +53,7 @@ value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -64,7 +64,7 @@ GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -76,7 +76,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -89,7 +89,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -98,7 +98,7 @@ value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -121,37 +121,37 @@ REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_NOP: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_INT: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TWO: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -160,19 +160,19 @@ res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _END_FOR: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_ITER: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -181,7 +181,7 @@ val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -506,13 +506,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -556,13 +556,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -606,13 +606,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -675,7 +675,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyFloat_Type); @@ -684,7 +684,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -727,7 +727,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyFloat_Type); @@ -736,7 +736,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -779,7 +779,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyFloat_Type); @@ -788,7 +788,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -797,7 +797,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -806,7 +806,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -815,7 +815,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -858,13 +858,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -889,7 +889,7 @@ } GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -902,7 +902,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -922,13 +922,13 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SLICE: { stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -937,7 +937,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -946,7 +946,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -955,7 +955,7 @@ res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1004,7 +1004,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1033,7 +1033,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1042,7 +1042,7 @@ getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1052,43 +1052,43 @@ ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LIST_APPEND: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_ADD: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_LIST_INT: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_SUBSCR: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1104,7 +1104,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1114,7 +1114,7 @@ retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; PyCodeObject *returning_code = get_code_with_logging(this_instr); if (returning_code == NULL) { @@ -1136,7 +1136,7 @@ res = temp; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1152,7 +1152,7 @@ awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1179,7 +1179,7 @@ retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; PyCodeObject *returning_code = get_code_with_logging(this_instr); if (returning_code == NULL) { @@ -1195,13 +1195,13 @@ value = temp; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_EXCEPT: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1210,7 +1210,7 @@ value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1219,13 +1219,13 @@ bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_NAME: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1243,7 +1243,7 @@ values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1257,7 +1257,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1270,7 +1270,7 @@ values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1281,7 +1281,7 @@ values[_i] = sym_new_not_null(ctx); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1296,25 +1296,25 @@ values[i] = sym_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_ATTR: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_GLOBAL: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1327,7 +1327,7 @@ locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1338,7 +1338,7 @@ v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1347,7 +1347,7 @@ res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1362,7 +1362,7 @@ REPLACE_OP(this_instr, _NOP, 0, 0); } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1428,7 +1428,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1464,7 +1464,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1492,13 +1492,13 @@ value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_DEREF: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1511,7 +1511,7 @@ str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1520,7 +1520,7 @@ interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1529,7 +1529,7 @@ template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1540,7 +1540,7 @@ tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1549,19 +1549,19 @@ list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LIST_EXTEND: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_UPDATE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1570,7 +1570,7 @@ set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1579,7 +1579,7 @@ map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1589,19 +1589,19 @@ case _DICT_UPDATE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DICT_MERGE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MAP_ADD: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1610,7 +1610,7 @@ attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1622,7 +1622,7 @@ stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1639,7 +1639,7 @@ self_or_null[0] = sym_new_unknown(ctx); } stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1781,19 +1781,19 @@ case _STORE_ATTR_INSTANCE_VALUE: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_WITH_HINT: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_SLOT: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1842,7 +1842,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } if (oparg & 16) { @@ -1853,7 +1853,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1893,13 +1893,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1943,13 +1943,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1991,13 +1991,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2006,7 +2006,7 @@ b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2044,13 +2044,13 @@ } stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2070,7 +2070,7 @@ b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2079,7 +2079,7 @@ b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2105,7 +2105,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2114,7 +2114,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2151,13 +2151,13 @@ len = sym_new_const(ctx, temp); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += -1; } stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2166,7 +2166,7 @@ attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2175,7 +2175,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2184,7 +2184,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2193,7 +2193,7 @@ values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2213,7 +2213,7 @@ stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2231,7 +2231,7 @@ next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2254,7 +2254,7 @@ next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2279,7 +2279,7 @@ next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2298,7 +2298,7 @@ next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2308,7 +2308,7 @@ ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2320,7 +2320,7 @@ method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2337,7 +2337,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2349,7 +2349,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2377,7 +2377,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2397,7 +2397,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2451,7 +2451,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2485,7 +2485,7 @@ new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2533,7 +2533,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2618,7 +2618,7 @@ } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2626,7 +2626,7 @@ JitOptRef new_frame; new_frame = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!CURRENT_FRAME_IS_INIT_SHIM()) { ctx->frame->stack_pointer = stack_pointer; } @@ -2705,7 +2705,7 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2731,7 +2731,7 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2757,7 +2757,7 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2799,13 +2799,13 @@ init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _EXIT_INIT_CHECK: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2814,7 +2814,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2823,7 +2823,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2832,7 +2832,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2841,7 +2841,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2874,13 +2874,13 @@ res = sym_new_const(ctx, temp); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += 2; } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2914,7 +2914,7 @@ } stack_pointer[-4] = res; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2931,7 +2931,7 @@ case _CALL_LIST_APPEND: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2940,7 +2940,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2949,7 +2949,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2958,7 +2958,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2967,7 +2967,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2990,7 +2990,7 @@ new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3015,7 +3015,7 @@ res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3037,7 +3037,7 @@ func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3058,7 +3058,7 @@ res = sym_new_unknown(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3067,7 +3067,7 @@ slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3090,7 +3090,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3102,7 +3102,7 @@ top = bottom; stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3141,7 +3141,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } bool lhs_int = sym_matches_type(lhs, &PyLong_Type); @@ -3179,7 +3179,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3225,7 +3225,7 @@ } sym_set_const(flag, Py_True); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3239,7 +3239,7 @@ } sym_set_const(flag, Py_False); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3257,7 +3257,7 @@ } sym_set_const(val, Py_None); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3274,7 +3274,7 @@ eliminate_pop_guard(this_instr, false); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3319,7 +3319,7 @@ value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3337,25 +3337,25 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_ONE: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_TWO: { stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3372,7 +3372,7 @@ value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3382,7 +3382,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3392,7 +3392,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3402,7 +3402,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3414,7 +3414,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3426,7 +3426,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 3a0e7e5d0d5..6519e8e4f3e 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -296,7 +296,7 @@ def _save_physical_sp(self, out: CWriter) -> None: diff = self.logical_sp - self.physical_sp out.start_line() out.emit(f"stack_pointer += {diff.to_c()};\n") - out.emit(f"assert(WITHIN_STACK_BOUNDS());\n") + out.emit(f"ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);\n") self.physical_sp = self.logical_sp self._print(out) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 857e926d119..0167f1b0ae5 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -86,6 +86,12 @@ do { \ #define TIER_TWO 2 +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + __attribute__((preserve_none)) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { From c0c65141b37029bfb364094a6dfb4c75ebf8359e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 3 Dec 2025 15:48:44 -0500 Subject: [PATCH 391/417] gh-140482: Avoid changing terminal settings in test_pty (gh-142202) The previous test_spawn_doesnt_hang test had a few problems: * It would cause ENV CHANGED failures if other tests were running concurrently due to stty changes * Typing while the test was running could cause it to fail --- Lib/test/test_pty.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index a2018e86444..7e4f4828ce0 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -3,7 +3,6 @@ is_android, is_apple_mobile, is_wasm32, reap_children, verbose, warnings_helper ) from test.support.import_helper import import_module -from test.support.os_helper import TESTFN, unlink # Skip these tests if termios is not available import_module('termios') @@ -299,26 +298,27 @@ def test_master_read(self): @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_spawn_doesnt_hang(self): - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'wb') as f: - STDOUT_FILENO = 1 - dup_stdout = os.dup(STDOUT_FILENO) - os.dup2(f.fileno(), STDOUT_FILENO) - buf = b'' - def master_read(fd): - nonlocal buf - data = os.read(fd, 1024) - buf += data - return data + # gh-140482: Do the test in a pty.fork() child to avoid messing + # with the interactive test runner's terminal settings. + pid, fd = pty.fork() + if pid == pty.CHILD: + pty.spawn([sys.executable, '-c', 'print("hi there")']) + os._exit(0) + + try: + buf = bytearray() try: - pty.spawn([sys.executable, '-c', 'print("hi there")'], - master_read) - finally: - os.dup2(dup_stdout, STDOUT_FILENO) - os.close(dup_stdout) - self.assertEqual(buf, b'hi there\r\n') - with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'hi there\r\n') + while (data := os.read(fd, 1024)) != b'': + buf.extend(data) + except OSError as e: + if e.errno != errno.EIO: + raise + + (pid, status) = os.waitpid(pid, 0) + self.assertEqual(status, 0) + self.assertEqual(buf.take_bytes(), b"hi there\r\n") + finally: + os.close(fd) class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" From 618dc367146069f8f0aaeb0a4a7f1b834dc4a213 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Wed, 3 Dec 2025 23:08:51 +0100 Subject: [PATCH 392/417] GH-142050: Jit stencils on Windows contain debug data (#142052) Co-authored-by: Savannah Ostrowski --- .../next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst | 1 + Tools/jit/_schema.py | 1 + Tools/jit/_targets.py | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst diff --git a/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst b/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst new file mode 100644 index 00000000000..8917d5df76e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst @@ -0,0 +1 @@ +Fixed a bug where JIT stencils produced on Windows contained debug data. Patch by Chris Eibl. diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index c47e9af924a..4e86abe6049 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -89,6 +89,7 @@ class COFFSection(typing.TypedDict): Characteristics: dict[ typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] ] + Name: dict[typing.Literal["Value"], str] Number: int RawDataSize: int Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index a76d8ff2792..4c188d74a68 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -267,6 +267,10 @@ class _COFF( def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: + name = section["Name"]["Value"] + if name == ".debug$S": + # skip debug sections + return flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} if "SectionData" in section: section_data_bytes = section["SectionData"]["Bytes"] From 547d8daf780646e2800bec598ed32085817c8606 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 3 Dec 2025 18:37:35 -0500 Subject: [PATCH 393/417] gh-142218: Fix split table dictionary crash (gh-142229) This fixes a regression introduced in gh-140558. The interpreter would crash if we inserted a non `str` key into a split table that matches an existing key. --- Lib/test/test_dict.py | 8 ++++++++ .../2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst | 2 ++ Objects/dictobject.c | 10 +++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 2e6c2bbdf19..665b3e843dd 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1621,6 +1621,14 @@ def __eq__(self, other): self.assertEqual(len(d), 1) + def test_split_table_update_with_str_subclass(self): + class MyStr(str): pass + class MyClass: pass + obj = MyClass() + obj.attr = 1 + obj.__dict__[MyStr('attr')] = 2 + self.assertEqual(obj.attr, 2) + class CAPITest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst new file mode 100644 index 00000000000..a8ce0fc6526 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst @@ -0,0 +1,2 @@ +Fix crash when inserting into a split table dictionary with a non +:class:`str` key that matches an existing key. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ee1c173ae4a..e0eef7b46df 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1914,10 +1914,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (old_value != value) { _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); - assert(!_PyDict_HasSplitTable(mp)); if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; From c5252045d3a7164f1829503d122091b5e469fda3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 3 Dec 2025 15:42:10 -0800 Subject: [PATCH 394/417] Being more flexible in when not to explicitly set the sysroot when compiling for WASI (GH-142242) --- Tools/wasm/wasi/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 06903fd25ab..d95cc99c8ea 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -271,7 +271,7 @@ def wasi_sdk_env(context): for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + if not wasi_sdk_path.name.startswith("wasi-sdk"): for compiler in ["CC", "CPP", "CXX"]: env[compiler] += f" --sysroot={sysroot}" From 1a7824a927f0706300af7bfc182884a43e2f587a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 3 Dec 2025 22:14:25 -0500 Subject: [PATCH 395/417] gh-141004: Add a CI job ensuring that new C APIs include documentation (GH-142102) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/CODEOWNERS | 3 + .github/workflows/build.yml | 3 + Makefile.pre.in | 5 + Tools/check-c-api-docs/ignored_c_api.txt | 93 +++++++++++ Tools/check-c-api-docs/main.py | 193 +++++++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 Tools/check-c-api-docs/ignored_c_api.txt create mode 100644 Tools/check-c-api-docs/main.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1086b426204..6acc156ebff 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -126,6 +126,9 @@ Doc/howto/clinic.rst @erlend-aasland @AA-Turner # C Analyser Tools/c-analyzer/ @ericsnowcurrently +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + # Fuzzing Modules/_xxtestfuzz/ @ammaraskar diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e15400e497..3d889fa128e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,6 +142,9 @@ jobs: - name: Check for unsupported C global variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + - name: Check for undocumented C APIs + run: make check-c-api-docs + build-windows: name: >- diff --git a/Makefile.pre.in b/Makefile.pre.in index 086adbdf262..f3086ec1462 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3322,6 +3322,11 @@ check-c-globals: --format summary \ --traceback +# Check for undocumented C APIs. +.PHONY: check-c-api-docs +check-c-api-docs: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/check-c-api-docs/main.py + # Find files with funny names .PHONY: funny funny: diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt new file mode 100644 index 00000000000..e81ffd51e19 --- /dev/null +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -0,0 +1,93 @@ +# pydtrace_probes.h +PyDTrace_AUDIT +PyDTrace_FUNCTION_ENTRY +PyDTrace_FUNCTION_RETURN +PyDTrace_GC_DONE +PyDTrace_GC_START +PyDTrace_IMPORT_FIND_LOAD_DONE +PyDTrace_IMPORT_FIND_LOAD_START +PyDTrace_INSTANCE_DELETE_DONE +PyDTrace_INSTANCE_DELETE_START +PyDTrace_INSTANCE_NEW_DONE +PyDTrace_INSTANCE_NEW_START +PyDTrace_LINE +# fileobject.h +Py_FileSystemDefaultEncodeErrors +Py_FileSystemDefaultEncoding +Py_HasFileSystemDefaultEncoding +Py_UTF8Mode +# pyhash.h +Py_HASH_EXTERNAL +# exports.h +PyAPI_DATA +Py_EXPORTED_SYMBOL +Py_IMPORTED_SYMBOL +Py_LOCAL_SYMBOL +# modsupport.h +PyABIInfo_FREETHREADING_AGNOSTIC +# moduleobject.h +PyModuleDef_Type +# object.h +Py_INVALID_SIZE +Py_TPFLAGS_HAVE_VERSION_TAG +Py_TPFLAGS_INLINE_VALUES +Py_TPFLAGS_IS_ABSTRACT +# pyexpat.h +PyExpat_CAPI_MAGIC +PyExpat_CAPSULE_NAME +# pyport.h +Py_ALIGNED +Py_ARITHMETIC_RIGHT_SHIFT +Py_CAN_START_THREADS +Py_FORCE_EXPANSION +Py_GCC_ATTRIBUTE +Py_LL +Py_SAFE_DOWNCAST +Py_ULL +Py_VA_COPY +# unicodeobject.h +Py_UNICODE_SIZE +# cpython/methodobject.h +PyCFunction_GET_CLASS +# cpython/compile.h +PyCF_ALLOW_INCOMPLETE_INPUT +PyCF_COMPILE_MASK +PyCF_DONT_IMPLY_DEDENT +PyCF_IGNORE_COOKIE +PyCF_MASK +PyCF_MASK_OBSOLETE +PyCF_SOURCE_IS_UTF8 +# cpython/descrobject.h +PyDescr_COMMON +PyDescr_NAME +PyDescr_TYPE +PyWrapperFlag_KEYWORDS +# cpython/fileobject.h +PyFile_NewStdPrinter +PyStdPrinter_Type +Py_UniversalNewlineFgets +# cpython/setobject.h +PySet_MINSIZE +# cpython/ceval.h +PyUnstable_CopyPerfMapFile +PyUnstable_PerfTrampoline_CompileCode +PyUnstable_PerfTrampoline_SetPersistAfterFork +# cpython/genobject.h +PyAsyncGenASend_CheckExact +# cpython/longintrepr.h +PyLong_BASE +PyLong_MASK +PyLong_SHIFT +# cpython/pyerrors.h +PyException_HEAD +# cpython/pyframe.h +PyUnstable_EXECUTABLE_KINDS +PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION +PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR +PyUnstable_EXECUTABLE_KIND_PY_FUNCTION +PyUnstable_EXECUTABLE_KIND_SKIP +# cpython/pylifecycle.h +Py_FrozenMain +# cpython/unicodeobject.h +PyUnicode_IS_COMPACT +PyUnicode_IS_COMPACT_ASCII diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py new file mode 100644 index 00000000000..6bdf80a9ae8 --- /dev/null +++ b/Tools/check-c-api-docs/main.py @@ -0,0 +1,193 @@ +import re +from pathlib import Path +import sys +import _colorize +import textwrap + +SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") +SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") +SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") +SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") + +CPYTHON = Path(__file__).parent.parent.parent +INCLUDE = CPYTHON / "Include" +C_API_DOCS = CPYTHON / "Doc" / "c-api" +IGNORED = ( + (CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") + .read_text() + .split("\n") +) + +for index, line in enumerate(IGNORED): + if line.startswith("#"): + IGNORED.pop(index) + +MISTAKE = """ +If this is a mistake and this script should not be failing, create an +issue and tag Peter (@ZeroIntensity) on it.\ +""" + + +def found_undocumented(singular: bool) -> str: + some = "an" if singular else "some" + s = "" if singular else "s" + these = "this" if singular else "these" + them = "it" if singular else "them" + were = "was" if singular else "were" + + return ( + textwrap.dedent( + f""" + Found {some} undocumented C API{s}! + + Python requires documentation on all public C API symbols, macros, and types. + If {these} API{s} {were} not meant to be public, prefix {them} with a + leading underscore (_PySomething_API) or move {them} to the internal C API + (pycore_*.h files). + + In exceptional cases, certain APIs can be ignored by adding them to + Tools/check-c-api-docs/ignored_c_api.txt + """ + ) + + MISTAKE + ) + + +def found_ignored_documented(singular: bool) -> str: + some = "a" if singular else "some" + s = "" if singular else "s" + them = "it" if singular else "them" + were = "was" if singular else "were" + they = "it" if singular else "they" + + return ( + textwrap.dedent( + f""" + Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but + {they} {were} found in the documentation. To fix this, remove {them} from + ignored_c_api.txt. + """ + ) + + MISTAKE + ) + + +def is_documented(name: str) -> bool: + """ + Is a name present in the C API documentation? + """ + for path in C_API_DOCS.iterdir(): + if path.is_dir(): + continue + if path.suffix != ".rst": + continue + + text = path.read_text(encoding="utf-8") + if name in text: + return True + + return False + + +def scan_file_for_docs(filename: str, text: str) -> tuple[list[str], list[str]]: + """ + Scan a header file for C API functions. + """ + undocumented: list[str] = [] + documented_ignored: list[str] = [] + colors = _colorize.get_colors() + + def check_for_name(name: str) -> None: + documented = is_documented(name) + if documented and (name in IGNORED): + documented_ignored.append(name) + elif not documented and (name not in IGNORED): + undocumented.append(name) + + for function in SIMPLE_FUNCTION_REGEX.finditer(text): + name = function.group(2) + if not name.startswith("Py"): + continue + + check_for_name(name) + + for macro in SIMPLE_MACRO_REGEX.finditer(text): + name = macro.group(1) + if not name.startswith("Py"): + continue + + if "(" in name: + name = name[: name.index("(")] + + check_for_name(name) + + for inline in SIMPLE_INLINE_REGEX.finditer(text): + name = inline.group(2) + if not name.startswith("Py"): + continue + + check_for_name(name) + + for data in SIMPLE_DATA_REGEX.finditer(text): + name = data.group(1) + if not name.startswith("Py"): + continue + + check_for_name(name) + + # Remove duplicates and sort alphabetically to keep the output deterministic + undocumented = list(set(undocumented)) + undocumented.sort() + + if undocumented or documented_ignored: + print(f"{filename} {colors.RED}BAD{colors.RESET}") + for name in undocumented: + print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") + for name in documented_ignored: + print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}") + else: + print(f"{filename} {colors.GREEN}OK{colors.RESET}") + + return undocumented, documented_ignored + + +def main() -> None: + print("Scanning for undocumented C API functions...") + files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] + all_missing: list[str] = [] + all_found_ignored: list[str] = [] + + for file in files: + if file.is_dir(): + continue + assert file.exists() + text = file.read_text(encoding="utf-8") + missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text) + all_found_ignored += ignored + all_missing += missing + + fail = False + to_check = [ + (all_missing, "missing", found_undocumented(len(all_missing) == 1)), + ( + all_found_ignored, + "documented but ignored", + found_ignored_documented(len(all_found_ignored) == 1), + ), + ] + for name_list, what, message in to_check: + if not name_list: + continue + + s = "s" if len(name_list) != 1 else "" + print(f"-- {len(name_list)} {what} C API{s} --") + for name in name_list: + print(f" - {name}") + print(message) + fail = True + + sys.exit(1 if fail else 0) + + +if __name__ == "__main__": + main() From fb404ab575de7ab7658069810b7d771fb70d9fe4 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 4 Dec 2025 09:33:15 +0000 Subject: [PATCH 396/417] gh-142225: Fix `PyABIInfo_VAR` macro (GH-142230) --- Include/modsupport.h | 2 +- .../next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst diff --git a/Include/modsupport.h b/Include/modsupport.h index 094b9ff0e5c..cb47ad8cd27 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -132,7 +132,7 @@ PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); ) \ ///////////////////////////////////////////////////////// -#define _PyABIInfo_DEFAULT() { \ +#define _PyABIInfo_DEFAULT { \ 1, 0, \ PyABIInfo_DEFAULT_FLAGS, \ PY_VERSION_HEX, \ diff --git a/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst b/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst new file mode 100644 index 00000000000..1eaf5b713d9 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst @@ -0,0 +1 @@ +Fixed the :c:macro:`PyABIInfo_VAR` macro. From 6825d5c11ddb1dac7602fde55f0ed64e1aab50e7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 4 Dec 2025 12:27:15 +0000 Subject: [PATCH 397/417] GH-139757: Fix reference leaks introduced in GH-140800 (GH-142257) --- Python/ceval.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 1709dda0cbe..46bf644106a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1228,6 +1228,8 @@ _Py_BuildString_StackRefSteal( goto cleanup; } res = _PyUnicode_JoinArray(&_Py_STR(empty), args_o, total_args); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); cleanup: // arguments is a pointer into the GC visible stack, // so we must NULL out values as we clear them. @@ -1239,8 +1241,6 @@ _Py_BuildString_StackRefSteal( return res; } - - PyObject * _Py_BuildMap_StackRefSteal( _PyStackRef *arguments, @@ -1257,6 +1257,8 @@ _Py_BuildMap_StackRefSteal( args_o+1, 2, half_args ); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); cleanup: // arguments is a pointer into the GC visible stack, // so we must NULL out values as we clear them. From 8392095bf969655faf785dd0932c3f02fc4ec311 Mon Sep 17 00:00:00 2001 From: Kir Chou <148194051+gkirchou@users.noreply.github.com> Date: Thu, 4 Dec 2025 22:32:23 +0900 Subject: [PATCH 398/417] gh-129483: Make `TestLocalTimeDisambiguation`'s time format locale independent (#142193) * Change to update %c to the exact time format. --------- Co-authored-by: Kir Chou --- Lib/test/datetimetester.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 7df27206206..ace56aab7ac 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -6300,21 +6300,21 @@ def test_vilnius_1941_fromutc(self): gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:59:59 1941 MSK+0300') self.assertEqual(ldt.fold, 0) self.assertFalse(ldt.dst()) gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 1) self.assertTrue(ldt.dst()) gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Tue Jun 24 00:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 0) self.assertTrue(ldt.dst()) @@ -6324,22 +6324,22 @@ def test_vilnius_1941_toutc(self): ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 19:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 20:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 21:59:59 1941 UTC') ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 22:00:00 1941 UTC') def test_constructors(self): From 2dac9e6016c81abbefa4256253ff5c59b29378a7 Mon Sep 17 00:00:00 2001 From: Alper Date: Thu, 4 Dec 2025 06:21:51 -0800 Subject: [PATCH 399/417] gh-116738: Statically initialize special constants in cmath module (gh-142161) The initialization during `mod_exec` wasn't thread-safe with multiple interpreters. --- ...-12-01-10-03-08.gh-issue-116738.972YsG.rst | 2 + Modules/cmathmodule.c | 225 ++++++++---------- 2 files changed, 101 insertions(+), 126 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst new file mode 100644 index 00000000000..d6d9d02b017 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst @@ -0,0 +1,2 @@ +Fix :mod:`cmath` data race when initializing trigonometric tables with +subinterpreters. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index aee3e4f343d..65fbcf5cdaa 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -163,8 +163,15 @@ special_type(double d) raised. */ -static Py_complex acos_special_values[7][7]; - +static Py_complex acos_special_values[7][7] = { + { {P34,INF}, {P,INF}, {P,INF}, {P,-INF}, {P,-INF}, {P34,-INF}, {N,INF} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P14,INF}, {0.,INF}, {0.,INF}, {0.,-INF}, {0.,-INF}, {P14,-INF}, {N,INF} }, + { {N,INF}, {N,N}, {N,N}, {N,N}, {N,N}, {N,-INF}, {N,N} } +}; /*[clinic input] cmath.acos -> Py_complex_protected @@ -202,7 +209,15 @@ cmath_acos_impl(PyObject *module, Py_complex z) } -static Py_complex acosh_special_values[7][7]; +static Py_complex acosh_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.acosh = cmath.acos @@ -257,7 +272,15 @@ cmath_asin_impl(PyObject *module, Py_complex z) } -static Py_complex asinh_special_values[7][7]; +static Py_complex asinh_special_values[7][7] = { + { {-INF,-P14}, {-INF,-0.}, {-INF,-0.}, {-INF,0.}, {-INF,0.}, {-INF,P14}, {-INF,N} }, + { {-INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-INF,P12}, {N,N} }, + { {-INF,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.asinh = cmath.acos @@ -318,7 +341,15 @@ cmath_atan_impl(PyObject *module, Py_complex z) } -static Py_complex atanh_special_values[7][7]; +static Py_complex atanh_special_values[7][7] = { + { {-0.,-P12}, {-0.,-P12}, {-0.,-P12}, {-0.,P12}, {-0.,P12}, {-0.,P12}, {-0.,N} }, + { {-0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-0.,P12}, {N,N} }, + { {-0.,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.,P12}, {-0.,N} }, + { {0.,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {0.,P12}, {N,N} }, + { {0.,-P12}, {0.,-P12}, {0.,-P12}, {0.,P12}, {0.,P12}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {N,N}, {N,N}, {N,N}, {N,N}, {0.,P12}, {N,N} } +}; /*[clinic input] cmath.atanh = cmath.acos @@ -391,7 +422,15 @@ cmath_cos_impl(PyObject *module, Py_complex z) /* cosh(infinity + i*y) needs to be dealt with specially */ -static Py_complex cosh_special_values[7][7]; +static Py_complex cosh_special_values[7][7] = { + { {INF,N}, {U,U}, {INF,0.}, {INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,0.}, {U,U}, {1.,0.}, {1.,-0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.cosh = cmath.acos @@ -453,7 +492,15 @@ cmath_cosh_impl(PyObject *module, Py_complex z) /* exp(infinity + i*y) and exp(-infinity + i*y) need special treatment for finite y */ -static Py_complex exp_special_values[7][7]; +static Py_complex exp_special_values[7][7] = { + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.exp = cmath.acos @@ -512,7 +559,15 @@ cmath_exp_impl(PyObject *module, Py_complex z) return r; } -static Py_complex log_special_values[7][7]; +static Py_complex log_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-P}, {-INF,P}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; static Py_complex c_log(Py_complex z) @@ -628,7 +683,15 @@ cmath_sin_impl(PyObject *module, Py_complex z) /* sinh(infinity + i*y) needs to be dealt with specially */ -static Py_complex sinh_special_values[7][7]; +static Py_complex sinh_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {0.,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.sinh = cmath.acos @@ -687,7 +750,15 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } -static Py_complex sqrt_special_values[7][7]; +static Py_complex sqrt_special_values[7][7] = { + { {INF,-INF}, {0.,-INF}, {0.,-INF}, {0.,INF}, {0.,INF}, {INF,INF}, {N,INF} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,INF}, {INF,N} }, + { {INF,-INF}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,INF}, {N,N} } +}; /*[clinic input] cmath.sqrt = cmath.acos @@ -786,7 +857,15 @@ cmath_tan_impl(PyObject *module, Py_complex z) /* tanh(infinity + i*y) needs to be dealt with specially */ -static Py_complex tanh_special_values[7][7]; +static Py_complex tanh_special_values[7][7] = { + { {-1.,0.}, {U,U}, {-1.,-0.}, {-1.,0.}, {U,U}, {-1.,0.}, {-1.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {-0.0,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.0,N}, {-0.,N} }, + { {0.0,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.0,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {1.,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {1.,0.}, {1.,0.} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.tanh = cmath.acos @@ -969,7 +1048,15 @@ cmath_polar_impl(PyObject *module, Py_complex z) */ -static Py_complex rect_special_values[7][7]; +static Py_complex rect_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,0.}, {-INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,0.}, {U,U}, {-0.,0.}, {-0.,-0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.rect @@ -1202,120 +1289,6 @@ cmath_exec(PyObject *mod) return -1; } - /* initialize special value tables */ - -#define INIT_SPECIAL_VALUES(NAME, BODY) { Py_complex* p = (Py_complex*)NAME; BODY } -#define C(REAL, IMAG) p->real = REAL; p->imag = IMAG; ++p; - - INIT_SPECIAL_VALUES(acos_special_values, { - C(P34,INF) C(P,INF) C(P,INF) C(P,-INF) C(P,-INF) C(P34,-INF) C(N,INF) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P14,INF) C(0.,INF) C(0.,INF) C(0.,-INF) C(0.,-INF) C(P14,-INF) C(N,INF) - C(N,INF) C(N,N) C(N,N) C(N,N) C(N,N) C(N,-INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(acosh_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(asinh_special_values, { - C(-INF,-P14) C(-INF,-0.) C(-INF,-0.) C(-INF,0.) C(-INF,0.) C(-INF,P14) C(-INF,N) - C(-INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-INF,P12) C(N,N) - C(-INF,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(atanh_special_values, { - C(-0.,-P12) C(-0.,-P12) C(-0.,-P12) C(-0.,P12) C(-0.,P12) C(-0.,P12) C(-0.,N) - C(-0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-0.,P12) C(N,N) - C(-0.,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.,P12) C(-0.,N) - C(0.,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,P12) C(0.,N) - C(0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(0.,P12) C(N,N) - C(0.,-P12) C(0.,-P12) C(0.,-P12) C(0.,P12) C(0.,P12) C(0.,P12) C(0.,N) - C(0.,-P12) C(N,N) C(N,N) C(N,N) C(N,N) C(0.,P12) C(N,N) - }) - - INIT_SPECIAL_VALUES(cosh_special_values, { - C(INF,N) C(U,U) C(INF,0.) C(INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,0.) C(U,U) C(1.,0.) C(1.,-0.) C(U,U) C(N,0.) C(N,0.) - C(N,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,0.) C(N,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(exp_special_values, { - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(log_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-P) C(-INF,P) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sinh_special_values, { - C(INF,N) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(0.,N) C(0.,N) - C(0.,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sqrt_special_values, { - C(INF,-INF) C(0.,-INF) C(0.,-INF) C(0.,INF) C(0.,INF) C(INF,INF) C(N,INF) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,INF) C(INF,N) - C(INF,-INF) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(tanh_special_values, { - C(-1.,0.) C(U,U) C(-1.,-0.) C(-1.,0.) C(U,U) C(-1.,0.) C(-1.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(-0.0,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.0,N) C(-0.,N) - C(0.0,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.0,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(1.,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(1.,0.) C(1.,0.) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(rect_special_values, { - C(INF,N) C(U,U) C(-INF,0.) C(-INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,0.) C(U,U) C(-0.,0.) C(-0.,-0.) C(U,U) C(0.,0.) C(0.,0.) - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) return 0; } From b3bf2128989e550a7a02acbaa47389023b2c6bc9 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 5 Dec 2025 04:28:08 +0800 Subject: [PATCH 400/417] gh-141976: Check stack bounds in JIT optimizer (GH-142201) --- Lib/test/test_generated_cases.py | 4 + ...-12-02-21-11-46.gh-issue-141976.yu7pDV.rst | 1 + Python/optimizer_analysis.c | 25 ++- Python/optimizer_cases.c.h | 173 ++++++++++++++++++ Tools/cases_generator/optimizer_generator.py | 2 +- Tools/cases_generator/stack.py | 13 +- 6 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ac62e11c274..de0dbab480f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2115,6 +2115,7 @@ def test_validate_uop_unused_input(self): """ output = """ case OP: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2132,6 +2133,7 @@ def test_validate_uop_unused_input(self): """ output = """ case OP: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2153,6 +2155,7 @@ def test_validate_uop_unused_output(self): case OP: { JitOptRef foo; foo = NULL; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = foo; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2172,6 +2175,7 @@ def test_validate_uop_unused_output(self): """ output = """ case OP: { + CHECK_STACK_BOUNDS(1); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst new file mode 100644 index 00000000000..f77315b7c37 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst @@ -0,0 +1 @@ +Check against abstract stack overflow in the JIT optimizer. diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 685659ef7c4..51722556554 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -144,10 +144,6 @@ incorrect_keys(PyObject *obj, uint32_t version) #define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) -#define WITHIN_STACK_BOUNDS() \ - (CURRENT_FRAME_IS_INIT_SHIM() || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) - - #define GETLOCAL(idx) ((ctx->frame->locals[idx])) #define REPLACE_OP(INST, OP, ARG, OPERAND) \ @@ -192,6 +188,27 @@ incorrect_keys(PyObject *obj, uint32_t version) #define JUMP_TO_LABEL(label) goto label; +static int +check_stack_bounds(JitOptContext *ctx, JitOptRef *stack_pointer, int offset, int opcode) +{ + int stack_level = (int)(stack_pointer + (offset) - ctx->frame->stack); + int should_check = !CURRENT_FRAME_IS_INIT_SHIM() || + (opcode == _RETURN_VALUE) || + (opcode == _RETURN_GENERATOR) || + (opcode == _YIELD_VALUE); + if (should_check && (stack_level < 0 || stack_level > STACK_SIZE())) { + ctx->contradiction = true; + ctx->done = true; + return 1; + } + return 0; +} + +#define CHECK_STACK_BOUNDS(offset) \ + if (check_stack_bounds(ctx, stack_pointer, offset, opcode)) { \ + break; \ + } \ + static int optimize_to_bool( _PyUOpInstruction *this_instr, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0c2d34d2e64..85bebed5867 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -33,6 +33,7 @@ if (sym_is_null(value)) { ctx->done = true; } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -42,6 +43,7 @@ case _LOAD_FAST: { JitOptRef value; value = GETLOCAL(oparg); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -51,6 +53,7 @@ case _LOAD_FAST_BORROW: { JitOptRef value; value = PyJitRef_Borrow(GETLOCAL(oparg)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -62,6 +65,7 @@ value = GETLOCAL(oparg); JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -74,6 +78,7 @@ PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -87,6 +92,7 @@ assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -97,6 +103,7 @@ JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -120,36 +127,42 @@ else if (typ == &PyUnicode_Type) { REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_NOP: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_INT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TWO: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -158,6 +171,7 @@ case _PUSH_NULL: { JitOptRef res; res = sym_new_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -165,12 +179,14 @@ } case _END_FOR: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_ITER: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -179,6 +195,7 @@ case _END_SEND: { JitOptRef val; val = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = val; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -504,12 +521,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -554,12 +573,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -604,12 +625,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -673,6 +696,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -682,6 +706,7 @@ if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -725,6 +750,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -734,6 +760,7 @@ if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -777,6 +804,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -786,6 +814,7 @@ if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -795,6 +824,7 @@ case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -804,6 +834,7 @@ case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -813,6 +844,7 @@ case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -856,12 +888,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -888,6 +922,7 @@ res = sym_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -900,6 +935,7 @@ case _BINARY_OP_EXTEND: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -920,6 +956,7 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -927,6 +964,7 @@ } case _STORE_SLICE: { + CHECK_STACK_BOUNDS(-4); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -935,6 +973,7 @@ case _BINARY_OP_SUBSCR_LIST_INT: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -944,6 +983,7 @@ case _BINARY_OP_SUBSCR_LIST_SLICE: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -953,6 +993,7 @@ case _BINARY_OP_SUBSCR_STR_INT: { JitOptRef res; res = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1002,6 +1043,7 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1031,6 +1073,7 @@ case _BINARY_OP_SUBSCR_DICT: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1040,6 +1083,7 @@ case _BINARY_OP_SUBSCR_CHECK_FUNC: { JitOptRef getitem; getitem = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = getitem; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1050,6 +1094,7 @@ JitOptRef new_frame; new_frame = PyJitRef_NULL; ctx->done = true; + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = new_frame; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1057,36 +1102,42 @@ } case _LIST_APPEND: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_ADD: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_LIST_INT: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_SUBSCR: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1102,6 +1153,7 @@ case _CALL_INTRINSIC_2: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1113,6 +1165,7 @@ JitOptRef res; retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; @@ -1134,6 +1187,7 @@ } stack_pointer = ctx->frame->stack_pointer; res = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1150,6 +1204,7 @@ case _GET_ANEXT: { JitOptRef awaitable; awaitable = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = awaitable; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1178,6 +1233,7 @@ JitOptRef value; retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; @@ -1193,6 +1249,7 @@ } stack_pointer = ctx->frame->stack_pointer; value = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1200,6 +1257,7 @@ } case _POP_EXCEPT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1208,6 +1266,7 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1217,6 +1276,7 @@ case _LOAD_BUILD_CLASS: { JitOptRef bc; bc = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = bc; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1224,6 +1284,7 @@ } case _STORE_NAME: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1242,6 +1303,7 @@ for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1254,6 +1316,7 @@ seq = stack_pointer[-1]; val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -1269,6 +1332,7 @@ for (int i = 0; i < oparg; i++) { values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1280,6 +1344,7 @@ for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1295,24 +1360,28 @@ for (int i = 0; i < totalargs; i++) { values[i] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS((oparg & 0xFF) + (oparg >> 8)); stack_pointer += (oparg & 0xFF) + (oparg >> 8); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_ATTR: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_GLOBAL: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1325,6 +1394,7 @@ case _LOAD_LOCALS: { JitOptRef locals; locals = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = locals; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1336,6 +1406,7 @@ case _LOAD_NAME: { JitOptRef v; v = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1346,6 +1417,7 @@ JitOptRef *res; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1361,6 +1433,7 @@ else { REPLACE_OP(this_instr, _NOP, 0, 0); } + CHECK_STACK_BOUNDS((oparg & 1)); stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1426,6 +1499,7 @@ else { res = sym_new_const(ctx, cnst); } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1462,6 +1536,7 @@ else { res = sym_new_const(ctx, cnst); } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1490,6 +1565,7 @@ case _LOAD_DEREF: { JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1497,6 +1573,7 @@ } case _STORE_DEREF: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1509,6 +1586,7 @@ case _BUILD_STRING: { JitOptRef str; str = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1518,6 +1596,7 @@ case _BUILD_INTERPOLATION: { JitOptRef interpolation; interpolation = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - (oparg & 1)); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1527,6 +1606,7 @@ case _BUILD_TEMPLATE: { JitOptRef template; template = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = template; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1538,6 +1618,7 @@ JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1547,6 +1628,7 @@ case _BUILD_LIST: { JitOptRef list; list = sym_new_type(ctx, &PyList_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1554,12 +1636,14 @@ } case _LIST_EXTEND: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_UPDATE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1568,6 +1652,7 @@ case _BUILD_SET: { JitOptRef set; set = sym_new_type(ctx, &PySet_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1577,6 +1662,7 @@ case _BUILD_MAP: { JitOptRef map; map = sym_new_type(ctx, &PyDict_Type); + CHECK_STACK_BOUNDS(1 - oparg*2); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1588,18 +1674,21 @@ } case _DICT_UPDATE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DICT_MERGE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MAP_ADD: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1608,6 +1697,7 @@ case _LOAD_SUPER_ATTR_ATTR: { JitOptRef attr_st; attr_st = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = attr_st; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1619,6 +1709,7 @@ JitOptRef self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1638,6 +1729,7 @@ if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS((oparg&1)); stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1780,18 +1872,21 @@ } case _STORE_ATTR_INSTANCE_VALUE: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_WITH_HINT: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_SLOT: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1840,6 +1935,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1851,6 +1947,7 @@ else { res = _Py_uop_sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1891,12 +1988,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1941,12 +2040,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1989,12 +2090,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2004,6 +2107,7 @@ case _IS_OP: { JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2042,12 +2146,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2068,6 +2174,7 @@ case _CONTAINS_OP_SET: { JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2077,6 +2184,7 @@ case _CONTAINS_OP_DICT: { JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2103,6 +2211,7 @@ case _IMPORT_NAME: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2112,6 +2221,7 @@ case _IMPORT_FROM: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2149,12 +2259,14 @@ REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = len; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += -1; } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = len; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2164,6 +2276,7 @@ case _MATCH_CLASS: { JitOptRef attrs; attrs = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = attrs; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2173,6 +2286,7 @@ case _MATCH_MAPPING: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2182,6 +2296,7 @@ case _MATCH_SEQUENCE: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2191,6 +2306,7 @@ case _MATCH_KEYS: { JitOptRef values_or_none; values_or_none = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = values_or_none; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2210,6 +2326,7 @@ iter = sym_new_not_null(ctx); index_or_null = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -2229,6 +2346,7 @@ case _FOR_ITER_TIER_TWO: { JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2252,6 +2370,7 @@ case _ITER_NEXT_LIST_TIER_TWO: { JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2277,6 +2396,7 @@ case _ITER_NEXT_TUPLE: { JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2296,6 +2416,7 @@ case _ITER_NEXT_RANGE: { JitOptRef next; next = sym_new_type(ctx, &PyLong_Type); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2306,6 +2427,7 @@ JitOptRef gen_frame; gen_frame = PyJitRef_NULL; ctx->done = true; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = gen_frame; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2319,6 +2441,7 @@ method_and_self = &stack_pointer[-1]; method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; + CHECK_STACK_BOUNDS(1); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2335,6 +2458,7 @@ case _WITH_EXCEPT_START: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2346,6 +2470,7 @@ JitOptRef new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -2374,6 +2499,7 @@ _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -2394,6 +2520,7 @@ _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -2448,6 +2575,7 @@ _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -2483,6 +2611,7 @@ break; } new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2531,6 +2660,7 @@ case _CALL_NON_PY_GENERAL: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2616,6 +2746,7 @@ } else { new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2625,6 +2756,7 @@ case _PUSH_FRAME: { JitOptRef new_frame; new_frame = stack_pointer[-1]; + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!CURRENT_FRAME_IS_INIT_SHIM()) { @@ -2703,6 +2835,7 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2729,6 +2862,7 @@ else { res = sym_new_type(ctx, &PyUnicode_Type); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2755,6 +2889,7 @@ else { res = sym_new_type(ctx, &PyTuple_Type); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2797,6 +2932,7 @@ assert((this_instr + 1)->opcode == _PUSH_FRAME); PyCodeObject *co = get_code_with_logging((this_instr + 1)); init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2804,6 +2940,7 @@ } case _EXIT_INIT_CHECK: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2812,6 +2949,7 @@ case _CALL_BUILTIN_CLASS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2821,6 +2959,7 @@ case _CALL_BUILTIN_O: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2830,6 +2969,7 @@ case _CALL_BUILTIN_FAST: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2839,6 +2979,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2872,12 +3013,14 @@ 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += 2; } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2912,6 +3055,7 @@ sym_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } + CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2930,6 +3074,7 @@ } case _CALL_LIST_APPEND: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2938,6 +3083,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2947,6 +3093,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2956,6 +3103,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2965,6 +3113,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2988,6 +3137,7 @@ break; } new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + CHECK_STACK_BOUNDS(-2 - oparg); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3013,6 +3163,7 @@ case _CALL_KW_NON_PY: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2 - oparg); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3035,6 +3186,7 @@ case _SET_FUNCTION_ATTRIBUTE: { JitOptRef func_out; func_out = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = func_out; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3056,6 +3208,7 @@ } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3065,6 +3218,7 @@ case _BUILD_SLICE: { JitOptRef slice; slice = sym_new_type(ctx, &PySlice_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3088,6 +3242,7 @@ case _FORMAT_WITH_SPEC: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3100,6 +3255,7 @@ bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = top; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3139,6 +3295,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3177,6 +3334,7 @@ else { res = sym_new_type(ctx, &PyFloat_Type); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3224,6 +3382,7 @@ eliminate_pop_guard(this_instr, value != Py_True); } sym_set_const(flag, Py_True); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3238,6 +3397,7 @@ eliminate_pop_guard(this_instr, value != Py_False); } sym_set_const(flag, Py_False); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3256,6 +3416,7 @@ eliminate_pop_guard(this_instr, true); } sym_set_const(val, Py_None); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3273,6 +3434,7 @@ assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3317,6 +3479,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3335,6 +3498,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3342,18 +3506,21 @@ } case _POP_CALL: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_ONE: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_TWO: { + CHECK_STACK_BOUNDS(-4); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3370,6 +3537,7 @@ case _POP_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3380,6 +3548,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3390,6 +3559,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = value; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3400,6 +3570,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = value; stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3411,6 +3582,7 @@ JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; @@ -3423,6 +3595,7 @@ JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 41df073cf6d..ab0a90e234b 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -445,7 +445,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack() + stack = Stack(check_stack_bounds=True) write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 6519e8e4f3e..53499558aed 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -216,11 +216,12 @@ def array_or_scalar(var: StackItem | Local) -> str: return "array" if var.is_array() else "scalar" class Stack: - def __init__(self) -> None: + def __init__(self, check_stack_bounds: bool = False) -> None: self.base_offset = PointerOffset.zero() self.physical_sp = PointerOffset.zero() self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] + self.check_stack_bounds = check_stack_bounds def drop(self, var: StackItem, check_liveness: bool) -> None: self.logical_sp = self.logical_sp.pop(var) @@ -316,8 +317,17 @@ def save_variables(self, out: CWriter) -> None: self._print(out) var_offset = var_offset.push(var.item) + def stack_bound_check(self, out: CWriter) -> None: + if not self.check_stack_bounds: + return + if self.physical_sp != self.logical_sp: + diff = self.logical_sp - self.physical_sp + out.start_line() + out.emit(f"CHECK_STACK_BOUNDS({diff});\n") + def flush(self, out: CWriter) -> None: self._print(out) + self.stack_bound_check(out) self.save_variables(out) self._save_physical_sp(out) out.start_line() @@ -347,6 +357,7 @@ def copy(self) -> "Stack": other.physical_sp = self.physical_sp other.logical_sp = self.logical_sp other.variables = [var.copy() for var in self.variables] + other.check_stack_bounds = self.check_stack_bounds return other def __eq__(self, other: object) -> bool: From 128d31637e8bf23f086d2a09081525adeeb1f65a Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 4 Dec 2025 20:15:04 -0600 Subject: [PATCH 401/417] gh-141926: Do not unset `RUNSHARED` when cross-compiling (#141958) --- .../next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst | 4 ++++ configure | 4 ---- configure.ac | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst diff --git a/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst b/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst new file mode 100644 index 00000000000..dab79ba5cf9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst @@ -0,0 +1,4 @@ +``RUNSHARED`` is no longer cleared when cross-compiling. Previously, +``RUNSHARED`` was cleared when cross-compiling, which breaks PGO when using +``--enabled-shared`` on systems where the cross-compiled CPython is otherwise +executable (e.g., via transparent emulation). diff --git a/configure b/configure index 620878bb181..1561f7f4134 100755 --- a/configure +++ b/configure @@ -7808,10 +7808,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 printf "%s\n" "$LDLIBRARY" >&6; } -if test "$cross_compiling" = yes; then - RUNSHARED= -fi - # HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } diff --git a/configure.ac b/configure.ac index 8ef479fe320..f2a7319d22d 100644 --- a/configure.ac +++ b/configure.ac @@ -1639,10 +1639,6 @@ else # shared is disabled fi AC_MSG_RESULT([$LDLIBRARY]) -if test "$cross_compiling" = yes; then - RUNSHARED= -fi - # HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) if test -z "$HOSTRUNNER" From 53ec7c8fc07eb6958869638a0cad70c52ad6fcf5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 Dec 2025 20:04:42 -0800 Subject: [PATCH 402/417] gh-142214: Fix two regressions in dataclasses (#142223) --- Lib/dataclasses.py | 14 +++++++--- Lib/test/test_dataclasses/__init__.py | 28 +++++++++++++++++++ ...-12-03-06-12-39.gh-issue-142214.appYNZ.rst | 12 ++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3ccb7246928..730ced72998 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -550,7 +550,12 @@ def __annotate__(format, /): new_annotations = {} for k in annotation_fields: - new_annotations[k] = cls_annotations[k] + # gh-142214: The annotation may be missing in unusual dynamic cases. + # If so, just skip it. + try: + new_annotations[k] = cls_annotations[k] + except KeyError: + pass if return_type is not MISSING: if format == Format.STRING: @@ -1399,9 +1404,10 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): f.type = ann # Fix the class reference in the __annotate__ method - init_annotate = newcls.__init__.__annotate__ - if getattr(init_annotate, "__generated_by_dataclasses__", False): - _update_func_cell_for__class__(init_annotate, cls, newcls) + init = newcls.__init__ + if init_annotate := getattr(init, "__annotate__", None): + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) return newcls diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 513dd78c438..3b335429b98 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -927,6 +927,20 @@ class C: validate_class(C) + def test_incomplete_annotations(self): + # gh-142214 + @dataclass + class C: + "doc" # needed because otherwise we fetch the annotations at the wrong time + x: int + + C.__annotate__ = lambda _: {} + + self.assertEqual( + annotationlib.get_annotations(C.__init__), + {"return": None} + ) + def test_missing_default(self): # Test that MISSING works the same as a default not being # specified. @@ -2578,6 +2592,20 @@ def __init__(self, x: int) -> None: self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + def test_slots_true_init_false(self): + # Test that slots=True and init=False work together and + # that __annotate__ is not added to __init__. + + @dataclass(slots=True, init=False) + class F: + x: int + + f = F() + f.x = 10 + self.assertEqual(f.x, 10) + + self.assertFalse(hasattr(F.__init__, "__annotate__")) + def test_init_false_forwardref(self): # Test forward references in fields not required for __init__ annotations. diff --git a/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst b/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst new file mode 100644 index 00000000000..b87430ec1a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst @@ -0,0 +1,12 @@ +Fix two regressions in :mod:`dataclasses` in Python 3.14.1 related to +annotations. + +* An exception is no longer raised if ``slots=True`` is used and the + ``__init__`` method does not have an ``__annotate__`` attribute + (likely because ``init=False`` was used). + +* An exception is no longer raised if annotations are requested on the + ``__init__`` method and one of the fields is not present in the class + annotations. This can occur in certain dynamic scenarios. + +Patch by Jelle Zijlstra. From 4238a975d78a0cc8f1751cfc63b3030b94b46aa8 Mon Sep 17 00:00:00 2001 From: Sanyam Khurana <8039608+CuriousLearner@users.noreply.github.com> Date: Fri, 5 Dec 2025 07:18:54 -0500 Subject: [PATCH 403/417] gh-48752: Add readline.get_pre_input_hook() function (#141586) Add readline.get_pre_input_hook() to retrieve the current pre-input hook. This allows applications to save and restore the hook without overwriting user settings. --- Doc/library/readline.rst | 9 ++++++ Lib/test/test_readline.py | 18 ++++++++++++ ...5-11-15-11-10-16.gh-issue-48752.aB3xYz.rst | 3 ++ Modules/clinic/readline.c.h | 28 ++++++++++++++++++- Modules/readline.c | 21 ++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 75db832c546..780cc773403 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -246,6 +246,15 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. function:: get_pre_input_hook() + + Get the current pre-input hook function, or ``None`` if no pre-input hook + function has been set. This function only exists if Python was compiled + for a version of the library that supports it. + + .. versionadded:: next + + .. _readline-completion: Completion diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 45192fe5082..3982686dd10 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -413,6 +413,24 @@ def test_write_read_limited_history(self): # So, we've only tested that the read did not fail. # See TestHistoryManipulation for the full test. + @unittest.skipUnless(hasattr(readline, "get_pre_input_hook"), + "get_pre_input_hook not available") + def test_get_pre_input_hook(self): + # Save and restore the original hook to avoid side effects + original_hook = readline.get_pre_input_hook() + self.addCleanup(readline.set_pre_input_hook, original_hook) + + # Test that get_pre_input_hook returns None when no hook is set + readline.set_pre_input_hook(None) + self.assertIsNone(readline.get_pre_input_hook()) + + # Set a hook and verify we can retrieve it + def my_hook(): + pass + + readline.set_pre_input_hook(my_hook) + self.assertIs(readline.get_pre_input_hook(), my_hook) + @unittest.skipUnless(support.Py_GIL_DISABLED, 'these tests can only possibly fail with GIL disabled') class FreeThreadingTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst b/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst new file mode 100644 index 00000000000..37b91196658 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst @@ -0,0 +1,3 @@ +Add :func:`readline.get_pre_input_hook` function to retrieve the current +pre-input hook. This allows applications to save and restore the hook +without overwriting user settings. Patch by Sanyam Khurana. diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index 696475f7d00..dc9381e4b97 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -349,6 +349,28 @@ exit: #endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ +#if defined(HAVE_RL_PRE_INPUT_HOOK) + +PyDoc_STRVAR(readline_get_pre_input_hook__doc__, +"get_pre_input_hook($module, /)\n" +"--\n" +"\n" +"Get the current pre-input hook function."); + +#define READLINE_GET_PRE_INPUT_HOOK_METHODDEF \ + {"get_pre_input_hook", (PyCFunction)readline_get_pre_input_hook, METH_NOARGS, readline_get_pre_input_hook__doc__}, + +static PyObject * +readline_get_pre_input_hook_impl(PyObject *module); + +static PyObject * +readline_get_pre_input_hook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return readline_get_pre_input_hook_impl(module); +} + +#endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ + PyDoc_STRVAR(readline_get_completion_type__doc__, "get_completion_type($module, /)\n" "--\n" @@ -794,7 +816,11 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #define READLINE_SET_PRE_INPUT_HOOK_METHODDEF #endif /* !defined(READLINE_SET_PRE_INPUT_HOOK_METHODDEF) */ +#ifndef READLINE_GET_PRE_INPUT_HOOK_METHODDEF + #define READLINE_GET_PRE_INPUT_HOOK_METHODDEF +#endif /* !defined(READLINE_GET_PRE_INPUT_HOOK_METHODDEF) */ + #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=88d9812b6caa2102 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4bd95070973cd0e2 input=a9049054013a1b77]*/ diff --git a/Modules/readline.c b/Modules/readline.c index e89755b0cb4..cc84eb6229e 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -572,6 +572,26 @@ readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) return set_hook("pre_input_hook", &state->pre_input_hook, function); } + +/* Get pre-input hook */ + +/*[clinic input] +readline.get_pre_input_hook + +Get the current pre-input hook function. +[clinic start generated code]*/ + +static PyObject * +readline_get_pre_input_hook_impl(PyObject *module) +/*[clinic end generated code: output=ad56b77a8e8981ca input=fb1e1b1fbd94e4e5]*/ +{ + readlinestate *state = get_readline_state(module); + if (state->pre_input_hook == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(state->pre_input_hook); +} + #endif @@ -1074,6 +1094,7 @@ static struct PyMethodDef readline_methods[] = READLINE_SET_STARTUP_HOOK_METHODDEF #ifdef HAVE_RL_PRE_INPUT_HOOK READLINE_SET_PRE_INPUT_HOOK_METHODDEF + READLINE_GET_PRE_INPUT_HOOK_METHODDEF #endif #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER READLINE_CLEAR_HISTORY_METHODDEF From cac4b04973ea4cee80b775782453cddcd694635d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 5 Dec 2025 13:10:51 +0000 Subject: [PATCH 404/417] Fix disk space issues in Android CI (#142289) --- Android/android.py | 39 +++++++++++++++++++--------- Android/testbed/app/build.gradle.kts | 4 +-- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Android/android.py b/Android/android.py index 25bb4ca70b5..d1a10be776e 100755 --- a/Android/android.py +++ b/Android/android.py @@ -29,6 +29,7 @@ ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() ) +ENV_SCRIPT = ANDROID_DIR / "android-env.sh" TESTBED_DIR = ANDROID_DIR / "testbed" CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" @@ -129,12 +130,11 @@ def android_env(host): sysconfig_filename = next(sysconfig_files).name host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] - env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( f"set -eu; " f"HOST={host}; " f"PREFIX={prefix}; " - f". {env_script}; " + f". {ENV_SCRIPT}; " f"export", check=True, shell=True, capture_output=True, encoding='utf-8', ).stdout @@ -151,7 +151,7 @@ def android_env(host): env[key] = value if not env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" + raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n" + env_output) return env @@ -281,15 +281,30 @@ def clean_all(context): def setup_ci(): - # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ - if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux": - run( - ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], - input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', - text=True, - ) - run(["sudo", "udevadm", "control", "--reload-rules"]) - run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + if "GITHUB_ACTIONS" in os.environ: + # Enable emulator hardware acceleration + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + if platform.system() == "Linux": + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + # Free up disk space by deleting unused versions of the NDK + # (https://github.com/freakboy3742/pyspamsum/pull/108). + for line in ENV_SCRIPT.read_text().splitlines(): + if match := re.fullmatch(r"ndk_version=(.+)", line): + ndk_version = match[1] + break + else: + raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") + + for item in (android_home / "ndk").iterdir(): + if item.name[0].isdigit() and item.name != ndk_version: + delete_glob(item) def setup_sdk(): diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 4de628a279c..14d43d8c4d5 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -79,7 +79,7 @@ android { val androidEnvFile = file("../../android-env.sh").absoluteFile namespace = "org.python.testbed" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "org.python.testbed" @@ -92,7 +92,7 @@ android { } throw GradleException("Failed to find API level in $androidEnvFile") } - targetSdk = 34 + targetSdk = 35 versionCode = 1 versionName = "1.0" From 1d8f3ed2eba762e60a02ff87e782a5c7dcd0e77c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 16:22:38 +0200 Subject: [PATCH 405/417] gh-101100: Fix references to the set methods (GH-141857) --- Doc/c-api/set.rst | 2 +- Doc/library/stdtypes.rst | 210 +++++++++++++++++++----------------- Doc/reference/datamodel.rst | 2 +- Doc/whatsnew/2.3.rst | 4 +- 4 files changed, 113 insertions(+), 105 deletions(-) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027..09c0fb6b9c5 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 086da1a705c..3899e5b59d8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4826,7 +4826,7 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`add ` and :meth:`remove `. +like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no hash value and cannot be used as either a dictionary key or as an element of another set. The :class:`frozenset` type is immutable and :term:`hashable` --- @@ -4848,164 +4848,172 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other, /) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other, /) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other, /) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other, /) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other, /) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem, /) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem, /) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem, /) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() - Remove all elements from the set. + Remove all elements from the set. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. .. _typesmapping: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index ebadbc215a0..5f79c6fe8f5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -449,7 +449,7 @@ Sets These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as - :meth:`add `. + :meth:`~set.add`. Frozen sets diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce..f43692b3dce 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) From 706fdda8b360120a25b272898df40c8913381723 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 16:24:35 +0200 Subject: [PATCH 406/417] gh-141370: Fix undefined behavior when using Py_ABS() (GH-141548) Co-authored-by: Sergey B Kirpichev --- Include/pymacro.h | 6 ++++++ Lib/test/test_bytes.py | 11 +++++++++++ Lib/test/test_marshal.py | 5 +++++ Lib/test/test_memoryview.py | 19 +++++++++++++++++++ Python/marshal.c | 2 +- Python/pystrhex.c | 3 +-- 6 files changed, 43 insertions(+), 3 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 857cdf12db9..7ecce44a0d2 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -116,6 +116,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a6cf899fa51..a55ec6cf3b8 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -549,6 +549,17 @@ def test_hex_separator_basics(self): self.assertEqual(three_bytes.hex(':', 2), 'b9:01ef') self.assertEqual(three_bytes.hex(':', 1), 'b9:01:ef') self.assertEqual(three_bytes.hex('*', -2), 'b901*ef') + self.assertEqual(three_bytes.hex(sep=':', bytes_per_sep=2), 'b9:01ef') + self.assertEqual(three_bytes.hex(sep='*', bytes_per_sep=-2), 'b901*ef') + for bytes_per_sep in 3, -3, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef') + except OverflowError: + pass value = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' self.assertEqual(value.hex('.', 8), '7b7305000000776f.726c646902000000.730500000068656c.6c6f690100000030') diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 8b1fb0eba1f..662bdfccc79 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -43,6 +43,11 @@ def test_ints(self): for expected in (-n, n): self.helper(expected) n = n >> 1 + n = 1 << 100 + while n: + for expected in (-n, -n+1, n-1, n): + self.helper(expected) + n = n >> 1 def test_int64(self): # Simulate int marshaling with TYPE_INT64. diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 64f440f180b..1bd58eb6408 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -600,6 +600,25 @@ def test_memoryview_hex(self): m2 = m1[::-1] self.assertEqual(m2.hex(), '30' * 200000) + def test_memoryview_hex_separator(self): + x = bytes(range(97, 102)) + m1 = memoryview(x) + m2 = m1[::-1] + self.assertEqual(m2.hex(':'), '65:64:63:62:61') + self.assertEqual(m2.hex(':', 2), '65:6463:6261') + self.assertEqual(m2.hex(':', -2), '6564:6362:61') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') + for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + except OverflowError: + pass + def test_copy(self): m = memoryview(b'abc') with self.assertRaises(TypeError): diff --git a/Python/marshal.c b/Python/marshal.c index 8b56de65755..69d6dd7cf0f 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -310,7 +310,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p) } if (!long_export.digits) { int8_t sign = long_export.value < 0 ? -1 : 1; - uint64_t abs_value = Py_ABS(long_export.value); + uint64_t abs_value = _Py_ABS_CAST(uint64_t, long_export.value); uint64_t d = abs_value; long l = 0; diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 38484f5a7d4..af2f5c5dce5 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -42,8 +42,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, else { bytes_per_sep_group = 0; } - - unsigned int abs_bytes_per_sep = Py_ABS(bytes_per_sep_group); + unsigned int abs_bytes_per_sep = _Py_ABS_CAST(unsigned int, bytes_per_sep_group); Py_ssize_t resultlen = 0; if (bytes_per_sep_group && arglen > 0) { /* How many sep characters we'll be inserting. */ From 100c726d9895ef26d0d279ae585c0228c0d8529f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 18:09:20 +0200 Subject: [PATCH 407/417] Add explanation comments for tests for overlapped ZIP entries (GH-137152) --- Lib/test/test_zipfile/test_core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 1edb5dde998..6887a5e5cc4 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2531,6 +2531,10 @@ def test_decompress_without_3rd_party_library(self): @requires_zlib() def test_full_overlap_different_names(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the first + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' @@ -2560,6 +2564,10 @@ def test_full_overlap_different_names(self): @requires_zlib() def test_full_overlap_different_names2(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the second + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2591,6 +2599,8 @@ def test_full_overlap_different_names2(self): @requires_zlib() def test_full_overlap_same_name(self): + # The ZIP file contains two central directory entries with + # the same name which refer to the same local header. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2623,6 +2633,8 @@ def test_full_overlap_same_name(self): @requires_zlib() def test_quoted_overlap(self): + # The ZIP file contains two files. The second local header + # is contained in the range of the first file. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' @@ -2654,6 +2666,7 @@ def test_quoted_overlap(self): @requires_zlib() def test_overlap_with_central_dir(self): + # The local header offset is equal to the central directory offset. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' @@ -2668,11 +2681,15 @@ def test_overlap_with_central_dir(self): self.assertEqual(zi.header_offset, 0) self.assertEqual(zi.compress_size, 11) self.assertEqual(zi.file_size, 1033) + # Found central directory signature PK\x01\x02 instead of + # local header signature PK\x03\x04. with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number'): zipf.read('a') @requires_zlib() def test_overlap_with_archive_comment(self): + # The local header is written after the central directory, + # in the archive comment. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' From 4b145297301fbcb18461a4e933a4188b2515fad4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 5 Dec 2025 08:21:31 -0800 Subject: [PATCH 408/417] GH-139862: Remove `color` from HelpFormatter (#142274) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/argparse.py | 2 -- .../next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 55ecdadd8c9..41467707d39 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -166,7 +166,6 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=True, ): # default setting for width if width is None: @@ -174,7 +173,6 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, diff --git a/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst b/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst new file mode 100644 index 00000000000..2bee8881a75 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst @@ -0,0 +1 @@ +Remove ``color`` parameter from :class:`!argparse.HelpFormatter` constructor. Color is controlled by :class:`~argparse.ArgumentParser`. From 4085ff7b32f91bad7d821e5564d8565c5928f7d1 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 5 Dec 2025 08:47:50 -0800 Subject: [PATCH 409/417] GH-142267: Cache formatter to avoid repeated `_set_color` calls (#142268) --- Lib/argparse.py | 18 ++++++++++++++---- ...5-12-04-23-26-12.gh-issue-142267.yOM6fP.rst | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 41467707d39..10393b6a02b 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1568,8 +1568,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1763,8 +1763,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1919,6 +1919,9 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) @@ -2750,6 +2753,13 @@ def _get_formatter(self): formatter._set_color(self.color) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter + # ===================== # Help-printing methods # ===================== diff --git a/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst b/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst new file mode 100644 index 00000000000..f46e82105fc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst @@ -0,0 +1 @@ +Improve :mod:`argparse` performance by caching the formatter used for argument validation. From 59f247e43bc93c607a5793c220bcaafb712cf542 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 19:17:01 +0200 Subject: [PATCH 410/417] gh-115952: Fix a potential virtual memory allocation denial of service in pickle (GH-119204) Loading a small data which does not even involve arbitrary code execution could consume arbitrary large amount of memory. There were three issues: * PUT and LONG_BINPUT with large argument (the C implementation only). Since the memo is implemented in C as a continuous dynamic array, a single opcode can cause its resizing to arbitrary size. Now the sparsity of memo indices is limited. * BINBYTES, BINBYTES8 and BYTEARRAY8 with large argument. They allocated the bytes or bytearray object of the specified size before reading into it. Now they read very large data by chunks. * BINSTRING, BINUNICODE, LONG4, BINUNICODE8 and FRAME with large argument. They read the whole data by calling the read() method of the underlying file object, which usually allocates the bytes object of the specified size before reading into it. Now they read very large data by chunks. Also add comprehensive benchmark suite to measure performance and memory impact of chunked reading optimization in PR #119204. Features: - Normal mode: benchmarks legitimate pickles (time/memory metrics) - Antagonistic mode: tests malicious pickles (DoS protection) - Baseline comparison: side-by-side comparison of two Python builds - Support for truncated data and sparse memo attack vectors Co-Authored-By: Claude Sonnet 4.5 Co-authored-by: Gregory P. Smith --- Lib/pickle.py | 34 +- Lib/test/pickletester.py | 167 ++- Lib/test/test_pickle.py | 8 +- ...-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst | 7 + Modules/_pickle.c | 427 ++++--- Tools/picklebench/README.md | 232 ++++ Tools/picklebench/memory_dos_impact.py | 1069 +++++++++++++++++ 7 files changed, 1767 insertions(+), 177 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst create mode 100644 Tools/picklebench/README.md create mode 100755 Tools/picklebench/memory_dos_impact.py diff --git a/Lib/pickle.py b/Lib/pickle.py index 729c215514a..f3025776623 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -189,6 +189,11 @@ def __init__(self, value): __all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = (1 << 20) + + class _Framer: _FRAME_SIZE_MIN = 4 @@ -287,7 +292,7 @@ def read(self, n): "pickle exhausted before end of frame") return data else: - return self.file_read(n) + return self._chunked_file_read(n) def readline(self): if self.current_frame: @@ -302,11 +307,23 @@ def readline(self): else: return self.file_readline() + def _chunked_file_read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + b = self.file_read(cursize) + while cursize < size and len(b) == cursize: + delta = min(cursize, size - cursize) + b += self.file_read(delta) + cursize += delta + return b + def load_frame(self, frame_size): if self.current_frame and self.current_frame.read() != b'': raise UnpicklingError( "beginning of a new frame before end of current frame") - self.current_frame = io.BytesIO(self.file_read(frame_size)) + data = self._chunked_file_read(frame_size) + if len(data) < frame_size: + raise EOFError + self.current_frame = io.BytesIO(data) # Tools used for pickling. @@ -1496,12 +1513,17 @@ def load_binbytes8(self): dispatch[BINBYTES8[0]] = load_binbytes8 def load_bytearray8(self): - len, = unpack(' maxsize: + size, = unpack(' maxsize: raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size " "of %d bytes" % maxsize) - b = bytearray(len) - self.readinto(b) + cursize = min(size, _MIN_READ_BUF_SIZE) + b = bytearray(cursize) + if self.readinto(b) == cursize: + while cursize < size and len(b) == cursize: + delta = min(cursize, size - cursize) + b += self.read(delta) + cursize += delta self.append(b) dispatch[BYTEARRAY8[0]] = load_bytearray8 diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index e3663e44546..4e3468bfcde 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -74,6 +74,15 @@ def count_opcode(code, pickle): def identity(x): return x +def itersize(start, stop): + # Produce geometrical increasing sequence from start to stop + # (inclusively) for tests. + size = start + while size < stop: + yield size + size <<= 1 + yield stop + class UnseekableIO(io.BytesIO): def peek(self, *args): @@ -853,9 +862,8 @@ def assert_is_copy(self, obj, objcopy, msg=None): self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) - def check_unpickling_error(self, errors, data): - with self.subTest(data=data), \ - self.assertRaises(errors): + def check_unpickling_error_strict(self, errors, data): + with self.assertRaises(errors): try: self.loads(data) except BaseException as exc: @@ -864,6 +872,10 @@ def check_unpickling_error(self, errors, data): (data, exc.__class__.__name__, exc)) raise + def check_unpickling_error(self, errors, data): + with self.subTest(data=data): + self.check_unpickling_error_strict(errors, data) + def test_load_from_data0(self): self.assert_is_copy(self._testdata, self.loads(DATA0)) @@ -1150,6 +1162,155 @@ def test_negative_32b_binput(self): dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' self.check_unpickling_error(ValueError, dumped) + def test_too_large_put(self): + # Test that PUT with large id does not cause allocation of + # too large memo table. The C implementation uses a dict-based memo + # for sparse indices (when idx > memo_len * 2) instead of allocating + # a massive array. This test verifies large sparse indices work without + # causing memory exhaustion. + # + # The following simple pickle creates an empty list, memoizes it + # using a large index, then loads it back on the stack, builds + # a tuple containing 2 identical empty lists and returns it. + data = lambda n: (b'((lp' + str(n).encode() + b'\n' + + b'g' + str(n).encode() + b'\nt.') + # 0: ( MARK + # 1: ( MARK + # 2: l LIST (MARK at 1) + # 3: p PUT 1000000000000 + # 18: g GET 1000000000000 + # 33: t TUPLE (MARK at 0) + # 34: . STOP + for idx in [10**6, 10**9, 10**12]: + if idx > sys.maxsize: + continue + self.assertEqual(self.loads(data(idx)), ([],)*2) + + def test_too_large_long_binput(self): + # Test that LONG_BINPUT with large id does not cause allocation of + # too large memo table. The C implementation uses a dict-based memo + # for sparse indices (when idx > memo_len * 2) instead of allocating + # a massive array. This test verifies large sparse indices work without + # causing memory exhaustion. + # + # The following simple pickle creates an empty list, memoizes it + # using a large index, then loads it back on the stack, builds + # a tuple containing 2 identical empty lists and returns it. + data = lambda n: (b'(]r' + struct.pack(' sys.maxsize')) + + def test_truncated_large_binunicode8(self): + data = lambda size: b'\x8d' + struct.pack('input_len will be 0; this is intentional so that when - unpickling from a file, the "we've run out of data" code paths will trigger, - causing the Unpickler to go back to the file for more data. Use the returned - size to tell you how much data you can process. */ +/* Don't call it directly: use _Unpickler_ReadInto() */ static Py_ssize_t -_Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) -{ - PyObject *data; - Py_ssize_t read_size; - - assert(self->read != NULL); - - if (_Unpickler_SkipConsumed(self) < 0) - return -1; - - if (n == READ_WHOLE_LINE) { - data = PyObject_CallNoArgs(self->readline); - } - else { - PyObject *len; - /* Prefetch some data without advancing the file pointer, if possible */ - if (self->peek && n < PREFETCH) { - len = PyLong_FromSsize_t(PREFETCH); - if (len == NULL) - return -1; - data = _Pickle_FastCall(self->peek, len); - if (data == NULL) { - if (!PyErr_ExceptionMatches(PyExc_NotImplementedError)) - return -1; - /* peek() is probably not supported by the given file object */ - PyErr_Clear(); - Py_CLEAR(self->peek); - } - else { - read_size = _Unpickler_SetStringInput(self, data); - Py_DECREF(data); - if (read_size < 0) { - return -1; - } - - self->prefetched_idx = 0; - if (n <= read_size) - return n; - } - } - len = PyLong_FromSsize_t(n); - if (len == NULL) - return -1; - data = _Pickle_FastCall(self->read, len); - } - if (data == NULL) - return -1; - - read_size = _Unpickler_SetStringInput(self, data); - Py_DECREF(data); - return read_size; -} - -/* Don't call it directly: use _Unpickler_Read() */ -static Py_ssize_t -_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t n) -{ - Py_ssize_t num_read; - - *s = NULL; - if (self->next_read_idx > PY_SSIZE_T_MAX - n) { - PyErr_SetString(st->UnpicklingError, - "read would overflow (invalid bytecode)"); - return -1; - } - - /* This case is handled by the _Unpickler_Read() macro for efficiency */ - assert(self->next_read_idx + n > self->input_len); - - if (!self->read) - return bad_readline(st); - - /* Extend the buffer to satisfy desired size */ - num_read = _Unpickler_ReadFromFile(self, n); - if (num_read < 0) - return -1; - if (num_read < n) - return bad_readline(st); - *s = self->input_buffer; - self->next_read_idx = n; - return n; -} - -/* Read `n` bytes from the unpickler's data source, storing the result in `buf`. - * - * This should only be used for non-small data reads where potentially - * avoiding a copy is beneficial. This method does not try to prefetch - * more data into the input buffer. - * - * _Unpickler_Read() is recommended in most cases. - */ -static Py_ssize_t -_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, - Py_ssize_t n) +_Unpickler_ReadIntoFromFile(PickleState *state, UnpicklerObject *self, char *buf, + Py_ssize_t n) { assert(n != READ_WHOLE_LINE); - /* Read from available buffer data, if any */ - Py_ssize_t in_buffer = self->input_len - self->next_read_idx; - if (in_buffer > 0) { - Py_ssize_t to_read = Py_MIN(in_buffer, n); - memcpy(buf, self->input_buffer + self->next_read_idx, to_read); - self->next_read_idx += to_read; - buf += to_read; - n -= to_read; - if (n == 0) { - /* Entire read was satisfied from buffer */ - return n; - } - } - - /* Read from file */ - if (!self->read) { - /* We're unpickling memory, this means the input is truncated */ - return bad_readline(state); - } - if (_Unpickler_SkipConsumed(self) < 0) { - return -1; - } - if (!self->readinto) { /* readinto() not supported on file-like object, fall back to read() * and copy into destination buffer (bpo-39681) */ @@ -1435,6 +1311,163 @@ _Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, return n; } +/* If reading from a file, we need to only pull the bytes we need, since there + may be multiple pickle objects arranged contiguously in the same input + buffer. + + If `n` is READ_WHOLE_LINE, read a whole line. Otherwise, read up to `n` + bytes from the input stream/buffer. + + Update the unpickler's input buffer with the newly-read data. Returns -1 on + failure; on success, returns the number of bytes read from the file. + + On success, self->input_len will be 0; this is intentional so that when + unpickling from a file, the "we've run out of data" code paths will trigger, + causing the Unpickler to go back to the file for more data. Use the returned + size to tell you how much data you can process. */ +static Py_ssize_t +_Unpickler_ReadFromFile(PickleState *state, UnpicklerObject *self, Py_ssize_t n) +{ + PyObject *data; + Py_ssize_t read_size; + + assert(self->read != NULL); + + if (_Unpickler_SkipConsumed(self) < 0) + return -1; + + if (n == READ_WHOLE_LINE) { + data = PyObject_CallNoArgs(self->readline); + if (data == NULL) { + return -1; + } + } + else { + PyObject *len; + /* Prefetch some data without advancing the file pointer, if possible */ + if (self->peek && n < PREFETCH) { + len = PyLong_FromSsize_t(PREFETCH); + if (len == NULL) + return -1; + data = _Pickle_FastCall(self->peek, len); + if (data == NULL) { + if (!PyErr_ExceptionMatches(PyExc_NotImplementedError)) + return -1; + /* peek() is probably not supported by the given file object */ + PyErr_Clear(); + Py_CLEAR(self->peek); + } + else { + read_size = _Unpickler_SetStringInput(self, data); + Py_DECREF(data); + if (read_size < 0) { + return -1; + } + + self->prefetched_idx = 0; + if (n <= read_size) + return n; + } + } + Py_ssize_t cursize = Py_MIN(n, MIN_READ_BUF_SIZE); + len = PyLong_FromSsize_t(cursize); + if (len == NULL) + return -1; + data = _Pickle_FastCall(self->read, len); + if (data == NULL) { + return -1; + } + while (cursize < n) { + Py_ssize_t prevsize = cursize; + // geometrically double the chunk size to avoid CPU DoS + cursize += Py_MIN(cursize, n - cursize); + if (_PyBytes_Resize(&data, cursize) < 0) { + return -1; + } + if (_Unpickler_ReadIntoFromFile(state, self, + PyBytes_AS_STRING(data) + prevsize, cursize - prevsize) < 0) + { + Py_DECREF(data); + return -1; + } + } + } + + read_size = _Unpickler_SetStringInput(self, data); + Py_DECREF(data); + return read_size; +} + +/* Don't call it directly: use _Unpickler_Read() */ +static Py_ssize_t +_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t n) +{ + Py_ssize_t num_read; + + *s = NULL; + if (self->next_read_idx > PY_SSIZE_T_MAX - n) { + PyErr_SetString(st->UnpicklingError, + "read would overflow (invalid bytecode)"); + return -1; + } + + /* This case is handled by the _Unpickler_Read() macro for efficiency */ + assert(self->next_read_idx + n > self->input_len); + + if (!self->read) + return bad_readline(st); + + /* Extend the buffer to satisfy desired size */ + num_read = _Unpickler_ReadFromFile(st, self, n); + if (num_read < 0) + return -1; + if (num_read < n) + return bad_readline(st); + *s = self->input_buffer; + self->next_read_idx = n; + return n; +} + +/* Read `n` bytes from the unpickler's data source, storing the result in `buf`. + * + * This should only be used for non-small data reads where potentially + * avoiding a copy is beneficial. This method does not try to prefetch + * more data into the input buffer. + * + * _Unpickler_Read() is recommended in most cases. + */ +static Py_ssize_t +_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, + Py_ssize_t n) +{ + assert(n != READ_WHOLE_LINE); + + /* Read from available buffer data, if any */ + Py_ssize_t in_buffer = self->input_len - self->next_read_idx; + if (in_buffer > 0) { + Py_ssize_t to_read = Py_MIN(in_buffer, n); + memcpy(buf, self->input_buffer + self->next_read_idx, to_read); + self->next_read_idx += to_read; + buf += to_read; + n -= to_read; + if (n == 0) { + /* Entire read was satisfied from buffer */ + return n; + } + } + + /* Read from file */ + if (!self->read) { + /* We're unpickling memory, this means the input is truncated */ + return bad_readline(state); + } + if (_Unpickler_SkipConsumed(self) < 0) { + return -1; + } + + return _Unpickler_ReadIntoFromFile(state, self, buf, n); +} + /* Read `n` bytes from the unpickler's data source, storing the result in `*s`. This should be used for all data reads, rather than accessing the unpickler's @@ -1492,7 +1525,7 @@ _Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) if (!self->read) return bad_readline(state); - num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE); + num_read = _Unpickler_ReadFromFile(state, self, READ_WHOLE_LINE); if (num_read < 0) return -1; if (num_read == 0 || self->input_buffer[num_read - 1] != '\n') @@ -1525,12 +1558,35 @@ _Unpickler_ResizeMemoList(UnpicklerObject *self, size_t new_size) /* Returns NULL if idx is out of bounds. */ static PyObject * -_Unpickler_MemoGet(UnpicklerObject *self, size_t idx) +_Unpickler_MemoGet(PickleState *st, UnpicklerObject *self, size_t idx) { - if (idx >= self->memo_size) - return NULL; - - return self->memo[idx]; + PyObject *value; + if (idx < self->memo_size) { + value = self->memo[idx]; + if (value != NULL) { + return value; + } + } + if (self->memo_dict != NULL) { + PyObject *key = PyLong_FromSize_t(idx); + if (key == NULL) { + return NULL; + } + if (idx < self->memo_size) { + (void)PyDict_Pop(self->memo_dict, key, &value); + // Migrate dict entry to array for faster future access + self->memo[idx] = value; + } + else { + value = PyDict_GetItemWithError(self->memo_dict, key); + } + Py_DECREF(key); + if (value != NULL || PyErr_Occurred()) { + return value; + } + } + PyErr_Format(st->UnpicklingError, "Memo value not found at index %zd", idx); + return NULL; } /* Returns -1 (with an exception set) on failure, 0 on success. @@ -1541,6 +1597,27 @@ _Unpickler_MemoPut(UnpicklerObject *self, size_t idx, PyObject *value) PyObject *old_item; if (idx >= self->memo_size) { + if (idx > self->memo_len * 2) { + /* The memo keys are too sparse. Use a dict instead of + * a continuous array for the memo. */ + if (self->memo_dict == NULL) { + self->memo_dict = PyDict_New(); + if (self->memo_dict == NULL) { + return -1; + } + } + PyObject *key = PyLong_FromSize_t(idx); + if (key == NULL) { + return -1; + } + + if (PyDict_SetItem(self->memo_dict, key, value) < 0) { + Py_DECREF(key); + return -1; + } + Py_DECREF(key); + return 0; + } if (_Unpickler_ResizeMemoList(self, idx * 2) < 0) return -1; assert(idx < self->memo_size); @@ -1610,6 +1687,7 @@ _Unpickler_New(PyObject *module) self->memo = memo; self->memo_size = MEMO_SIZE; self->memo_len = 0; + self->memo_dict = NULL; self->persistent_load = NULL; self->persistent_load_attr = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); @@ -5582,13 +5660,28 @@ load_counted_binbytes(PickleState *state, UnpicklerObject *self, int nbytes) return -1; } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) - return -1; - if (_Unpickler_ReadInto(state, self, PyBytes_AS_STRING(bytes), size) < 0) { - Py_DECREF(bytes); + Py_ssize_t cursize = Py_MIN(size, MIN_READ_BUF_SIZE); + Py_ssize_t prevsize = 0; + bytes = PyBytes_FromStringAndSize(NULL, cursize); + if (bytes == NULL) { return -1; } + while (1) { + if (_Unpickler_ReadInto(state, self, + PyBytes_AS_STRING(bytes) + prevsize, cursize - prevsize) < 0) + { + Py_DECREF(bytes); + return -1; + } + if (cursize >= size) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, size - cursize); + if (_PyBytes_Resize(&bytes, cursize) < 0) { + return -1; + } + } PDATA_PUSH(self->stack, bytes, -1); return 0; @@ -5613,14 +5706,27 @@ load_counted_bytearray(PickleState *state, UnpicklerObject *self) return -1; } - bytearray = PyByteArray_FromStringAndSize(NULL, size); + Py_ssize_t cursize = Py_MIN(size, MIN_READ_BUF_SIZE); + Py_ssize_t prevsize = 0; + bytearray = PyByteArray_FromStringAndSize(NULL, cursize); if (bytearray == NULL) { return -1; } - char *str = PyByteArray_AS_STRING(bytearray); - if (_Unpickler_ReadInto(state, self, str, size) < 0) { - Py_DECREF(bytearray); - return -1; + while (1) { + if (_Unpickler_ReadInto(state, self, + PyByteArray_AS_STRING(bytearray) + prevsize, + cursize - prevsize) < 0) { + Py_DECREF(bytearray); + return -1; + } + if (cursize >= size) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, size - cursize); + if (PyByteArray_Resize(bytearray, cursize) < 0) { + return -1; + } } PDATA_PUSH(self->stack, bytearray, -1); @@ -6222,20 +6328,15 @@ load_get(PickleState *st, UnpicklerObject *self) if (key == NULL) return -1; idx = PyLong_AsSsize_t(key); + Py_DECREF(key); if (idx == -1 && PyErr_Occurred()) { - Py_DECREF(key); return -1; } - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - } - Py_DECREF(key); return -1; } - Py_DECREF(key); PDATA_APPEND(self->stack, value, -1); return 0; @@ -6253,13 +6354,8 @@ load_binget(PickleState *st, UnpicklerObject *self) idx = Py_CHARMASK(s[0]); - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - PyObject *key = PyLong_FromSsize_t(idx); - if (key != NULL) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - Py_DECREF(key); - } return -1; } @@ -6279,13 +6375,8 @@ load_long_binget(PickleState *st, UnpicklerObject *self) idx = calc_binsize(s, 4); - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - PyObject *key = PyLong_FromSsize_t(idx); - if (key != NULL) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - Py_DECREF(key); - } return -1; } @@ -7250,6 +7341,7 @@ Unpickler_clear(PyObject *op) self->buffer.buf = NULL; } + Py_CLEAR(self->memo_dict); _Unpickler_MemoCleanup(self); PyMem_Free(self->marks); self->marks = NULL; @@ -7286,6 +7378,7 @@ Unpickler_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(self->persistent_load); Py_VISIT(self->persistent_load_attr); Py_VISIT(self->buffers); + Py_VISIT(self->memo_dict); PyObject **memo = self->memo; if (memo) { Py_ssize_t i = self->memo_size; diff --git a/Tools/picklebench/README.md b/Tools/picklebench/README.md new file mode 100644 index 00000000000..7d52485c386 --- /dev/null +++ b/Tools/picklebench/README.md @@ -0,0 +1,232 @@ +# Pickle Chunked Reading Benchmark + +This benchmark measures the performance impact of the chunked reading optimization in GH PR #119204 for the pickle module. + +## What This Tests + +The PR adds chunked reading (1MB chunks) to prevent memory exhaustion when unpickling large objects: +- **BINBYTES8** - Large bytes objects (protocol 4+) +- **BINUNICODE8** - Large strings (protocol 4+) +- **BYTEARRAY8** - Large bytearrays (protocol 5) +- **FRAME** - Large frames +- **LONG4** - Large integers +- An antagonistic mode that tests using memory denial of service inducing malicious pickles. + +## Quick Start + +```bash +# Run full benchmark suite (1MiB → 200MiB, takes several minutes) +build/python Tools/picklebench/memory_dos_impact.py + +# Test just a few sizes (quick test: 1, 10, 50 MiB) +build/python Tools/picklebench/memory_dos_impact.py --sizes 1 10 50 + +# Test smaller range for faster results +build/python Tools/picklebench/memory_dos_impact.py --sizes 1 5 10 + +# Output as markdown for reports +build/python Tools/picklebench/memory_dos_impact.py --format markdown > results.md + +# Test with protocol 4 instead of 5 +build/python Tools/picklebench/memory_dos_impact.py --protocol 4 +``` + +**Note:** Sizes are specified in MiB. Use `--sizes 1 2 5` for 1MiB, 2MiB, 5MiB objects. + +## Antagonistic Mode (DoS Protection Test) + +The `--antagonistic` flag tests **malicious pickles** that demonstrate the memory DoS protection: + +```bash +# Quick DoS protection test (claims 10, 50, 100 MB but provides 1KB) +build/python Tools/picklebench/memory_dos_impact.py --antagonistic --sizes 10 50 100 + +# Full DoS test (default: 10, 50, 100, 500, 1000, 5000 MB claimed) +build/python Tools/picklebench/memory_dos_impact.py --antagonistic +``` + +### What This Tests + +Unlike normal benchmarks that test **legitimate pickles**, antagonistic mode tests: +- **Truncated BINBYTES8**: Claims 100MB but provides only 1KB (will fail to unpickle) +- **Truncated BINUNICODE8**: Same for strings +- **Truncated BYTEARRAY8**: Same for bytearrays +- **Sparse memo attacks**: PUT at index 1 billion (would allocate huge array before PR) + +**Key difference:** +- **Normal mode**: Tests real data, shows ~5% time overhead +- **Antagonistic mode**: Tests malicious data, shows ~99% memory savings + +### Expected Results + +``` +100MB Claimed (actual: 1KB) + binbytes8_100MB_claim + Peak memory: 1.00 MB (claimed: 100 MB, saved: 99.00 MB, 99.0%) + Error: UnpicklingError ← Expected! + +Summary: + Average claimed: 126.2 MB + Average peak: 0.54 MB + Average saved: 125.7 MB (99.6% reduction) +Protection Status: ✓ Memory DoS attacks mitigated by chunked reading +``` + +**Before PR**: Would allocate full claimed size (100MB+), potentially crash +**After PR**: Allocates 1MB chunks, fails fast with minimal memory + +This demonstrates the **security improvement** - protection against memory exhaustion attacks. + +## Before/After Comparison + +The benchmark includes an automatic comparison feature that runs the same tests on both a baseline and current Python build. + +### Option 1: Automatic Comparison (Recommended) + +Build both versions, then use `--baseline` to automatically compare: + +```bash +# Build the baseline (main branch without PR) +git checkout main +mkdir -p build-main +cd build-main && ../configure && make -j $(nproc) && cd .. + +# Build the current version (with PR) +git checkout unpickle-overallocate +mkdir -p build +cd build && ../configure && make -j $(nproc) && cd .. + +# Run automatic comparison (quick test with a few sizes) +build/python Tools/picklebench/memory_dos_impact.py \ + --baseline build-main/python \ + --sizes 1 10 50 + +# Full comparison (all default sizes) +build/python Tools/picklebench/memory_dos_impact.py \ + --baseline build-main/python +``` + +The comparison output shows: +- Side-by-side metrics (Current vs Baseline) +- Percentage change for time and memory +- Overall summary statistics + +### Interpreting Comparison Results + +- **Time change**: Small positive % is expected (chunking adds overhead, typically 5-10%) +- **Memory change**: Negative % is good (chunking saves memory, especially for large objects) +- **Trade-off**: Slightly slower but much safer against memory exhaustion attacks + +### Option 2: Manual Comparison + +Save results separately and compare manually: + +```bash +# Baseline results +build-main/python Tools/picklebench/memory_dos_impact.py --format json > baseline.json + +# Current results +build/python Tools/picklebench/memory_dos_impact.py --format json > current.json + +# Manual comparison +diff -y <(jq '.' baseline.json) <(jq '.' current.json) +``` + +## Understanding the Results + +### Critical Sizes + +The default test suite includes: +- **< 1MiB (999,000 bytes)**: No chunking, allocates full size upfront +- **= 1MiB (1,048,576 bytes)**: Threshold, chunking just starts +- **> 1MiB (1,048,577 bytes)**: Chunked reading engaged +- **1, 2, 5, 10MiB**: Show scaling behavior with chunking +- **20, 50, 100, 200MiB**: Stress test large object handling + +**Note:** The full suite may require more than 16GiB of RAM. + +### Key Metrics + +- **Time (mean)**: Average unpickling time - should be similar before/after +- **Time (stdev)**: Consistency - lower is better +- **Peak Memory**: Maximum memory during unpickling - **expected to be LOWER after PR** +- **Pickle Size**: Size of the serialized data on disk + +### Test Types + +| Test | What It Stresses | +|------|------------------| +| `bytes_*` | BINBYTES8 opcode, raw binary data | +| `string_ascii_*` | BINUNICODE8 with simple ASCII | +| `string_utf8_*` | BINUNICODE8 with multibyte UTF-8 (€ chars) | +| `bytearray_*` | BYTEARRAY8 opcode (protocol 5) | +| `list_large_items_*` | Multiple chunked reads in sequence | +| `dict_large_values_*` | Chunking in dict deserialization | +| `nested_*` | Realistic mixed data structures | +| `tuple_*` | Immutable structures | + +## Expected Results + +### Before PR (main branch) +- Single large allocation per object +- Risk of memory exhaustion with malicious pickles + +### After PR (unpickle-overallocate branch) +- Chunked allocation (1MB at a time) +- **Slightly higher CPU time** (multiple allocations + resizing) +- **Significantly lower peak memory** (no large pre-allocation) +- Protection against DoS via memory exhaustion + +## Advanced Usage + +### Test Specific Sizes + +```bash +# Test only 5MiB and 10MiB objects +build/python Tools/picklebench/memory_dos_impact.py --sizes 5 10 + +# Test large objects: 50, 100, 200 MiB +build/python Tools/picklebench/memory_dos_impact.py --sizes 50 100 200 +``` + +### More Iterations for Stable Timing + +```bash +# Run 10 iterations per test for better statistics +build/python Tools/picklebench/memory_dos_impact.py --iterations 10 --sizes 1 10 +``` + +### JSON Output for Analysis + +```bash +# Generate JSON for programmatic analysis +build/python Tools/picklebench/memory_dos_impact.py --format json | python -m json.tool +``` + +## Interpreting Memory Results + +The **peak memory** metric shows the maximum memory allocated during unpickling: + +- **Without chunking**: Allocates full size immediately + - 10MB object → 10MB allocation upfront + +- **With chunking**: Allocates in 1MB chunks, grows geometrically + - 10MB object → starts with 1MB, grows: 2MB, 4MB, 8MB (final: ~10MB total) + - Peak is lower because allocation is incremental + +## Typical Results + +On a system with the PR applied, you should see: + +``` +1.00MiB Test Results + bytes_1.00MiB: ~0.3ms, 1.00MiB peak (just at threshold) + +2.00MiB Test Results + bytes_2.00MiB: ~0.8ms, 2.00MiB peak (chunked: 1MiB → 2MiB) + +10.00MiB Test Results + bytes_10.00MiB: ~3-5ms, 10.00MiB peak (chunked: 1→2→4→8→10 MiB) +``` + +Time overhead is minimal (~10-20% for very large objects), but memory safety is significantly improved. diff --git a/Tools/picklebench/memory_dos_impact.py b/Tools/picklebench/memory_dos_impact.py new file mode 100755 index 00000000000..3bad6586c46 --- /dev/null +++ b/Tools/picklebench/memory_dos_impact.py @@ -0,0 +1,1069 @@ +#!/usr/bin/env python3 +# +# Author: Claude Sonnet 4.5 as driven by gpshead +# +""" +Microbenchmark for pickle module chunked reading performance (GH PR #119204). + +This script generates Python data structures that act as antagonistic load +tests for the chunked reading code introduced to prevent memory exhaustion when +unpickling large objects. + +The PR adds chunked reading (1MB chunks) for: +- BINBYTES8 (large bytes) +- BINUNICODE8 (large strings) +- BYTEARRAY8 (large bytearrays) +- FRAME (large frames) +- LONG4 (large integers) + +Including an antagonistic mode that exercies memory denial of service pickles. + +Usage: + python memory_dos_impact.py --help +""" + +import argparse +import gc +import io +import json +import os +import pickle +import statistics +import struct +import subprocess +import sys +import tempfile +import tracemalloc +from pathlib import Path +from time import perf_counter +from typing import Any, Dict, List, Tuple, Optional + + +# Configuration +MIN_READ_BUF_SIZE = 1 << 20 # 1MB - matches pickle.py _MIN_READ_BUF_SIZE + +# Test sizes in MiB +DEFAULT_SIZES_MIB = [1, 2, 5, 10, 20, 50, 100, 200] + +# Convert to bytes, plus threshold boundary tests +DEFAULT_SIZES = ( + [999_000] # Below 1MiB (no chunking) + + [size * (1 << 20) for size in DEFAULT_SIZES_MIB] # MiB to bytes + + [1_048_577] # Just above 1MiB (minimal chunking overhead) +) +DEFAULT_SIZES.sort() + +# Baseline benchmark configuration +BASELINE_BENCHMARK_TIMEOUT_SECONDS = 600 # 10 minutes + +# Sparse memo attack test configuration +# Format: test_name -> (memo_index, baseline_memory_note) +SPARSE_MEMO_TESTS = { + "sparse_memo_1M": (1_000_000, "~8 MB array"), + "sparse_memo_100M": (100_000_000, "~800 MB array"), + "sparse_memo_1B": (1_000_000_000, "~8 GB array"), +} + + +# Utility functions + +def _extract_size_mb(size_key: str) -> float: + """Extract numeric MiB value from size_key like '10.00MB' or '1.00MiB'. + + Returns 0.0 for non-numeric keys (they'll be sorted last). + """ + try: + return float(size_key.replace('MB', '').replace('MiB', '')) + except ValueError: + return 999999.0 # Put non-numeric keys last + + +def _format_output(results: Dict[str, Dict[str, Any]], format_type: str, is_antagonistic: bool) -> str: + """Format benchmark results according to requested format. + + Args: + results: Benchmark results dictionary + format_type: Output format ('text', 'markdown', or 'json') + is_antagonistic: Whether these are antagonistic (DoS) test results + + Returns: + Formatted output string + """ + if format_type == 'json': + return Reporter.format_json(results) + elif is_antagonistic: + # Antagonistic mode uses specialized formatter for text/markdown + return Reporter.format_antagonistic(results) + elif format_type == 'text': + return Reporter.format_text(results) + elif format_type == 'markdown': + return Reporter.format_markdown(results) + else: + # Default to text format + return Reporter.format_text(results) + + +class AntagonisticGenerator: + """Generate malicious/truncated pickles for DoS protection testing. + + These pickles claim large sizes but provide minimal data, causing them to fail + during unpickling. They demonstrate the memory protection of chunked reading. + """ + + @staticmethod + def truncated_binbytes8(claimed_size: int, actual_size: int = 1024) -> bytes: + """BINBYTES8 claiming `claimed_size` but providing only `actual_size` bytes. + + This will fail with UnpicklingError but demonstrates peak memory usage. + Before PR: Allocates full claimed_size + After PR: Allocates in 1MB chunks, fails fast + """ + return b'\x8e' + struct.pack(' bytes: + """BINUNICODE8 claiming `claimed_size` but providing only `actual_size` bytes.""" + return b'\x8d' + struct.pack(' bytes: + """BYTEARRAY8 claiming `claimed_size` but providing only `actual_size` bytes.""" + return b'\x96' + struct.pack(' bytes: + """FRAME claiming `claimed_size` but providing minimal data.""" + return b'\x95' + struct.pack(' bytes: + """LONG_BINPUT with huge sparse index. + + Before PR: Tries to allocate array with `index` slots (OOM) + After PR: Uses dict-based memo for sparse indices + """ + return (b'(]r' + struct.pack(' bytes: + """Multiple BINBYTES8 claims in sequence. + + Tests that multiple large claims don't accumulate memory. + """ + data = b'(' # MARK + for _ in range(count): + data += b'\x8e' + struct.pack(' bytes: + """Generate random bytes of specified size.""" + return os.urandom(size) + + @staticmethod + def large_string_ascii(size: int) -> str: + """Generate ASCII string of specified size.""" + return 'x' * size + + @staticmethod + def large_string_multibyte(size: int) -> str: + """Generate multibyte UTF-8 string (3 bytes per char for €).""" + # Each € is 3 bytes in UTF-8 + return '€' * (size // 3) + + @staticmethod + def large_bytearray(size: int) -> bytearray: + """Generate bytearray of specified size.""" + return bytearray(os.urandom(size)) + + @staticmethod + def list_of_large_bytes(item_size: int, count: int) -> List[bytes]: + """Generate list containing multiple large bytes objects.""" + return [os.urandom(item_size) for _ in range(count)] + + @staticmethod + def dict_with_large_values(value_size: int, count: int) -> Dict[str, bytes]: + """Generate dict with large bytes values.""" + return { + f'key_{i}': os.urandom(value_size) + for i in range(count) + } + + @staticmethod + def nested_structure(size: int) -> Dict[str, Any]: + """Generate nested structure with various large objects.""" + chunk_size = size // 4 + return { + 'name': 'test_object', + 'data': { + 'bytes': os.urandom(chunk_size), + 'string': 's' * chunk_size, + 'bytearray': bytearray(b'b' * chunk_size), + }, + 'items': [os.urandom(chunk_size // 4) for _ in range(4)], + 'metadata': { + 'size': size, + 'type': 'nested', + }, + } + + @staticmethod + def tuple_of_large_objects(size: int) -> Tuple[bytes, str, bytearray]: + """Generate tuple with large objects (immutable, different pickle path).""" + chunk_size = size // 3 + return ( + os.urandom(chunk_size), + 'x' * chunk_size, + bytearray(b'y' * chunk_size), + ) + + +class PickleBenchmark: + """Benchmark pickle unpickling performance and memory usage.""" + + def __init__(self, obj: Any, protocol: int = 5, iterations: int = 3): + self.obj = obj + self.protocol = protocol + self.iterations = iterations + self.pickle_data = pickle.dumps(obj, protocol=protocol) + self.pickle_size = len(self.pickle_data) + + def benchmark_time(self) -> Dict[str, float]: + """Measure unpickling time over multiple iterations.""" + times = [] + + for _ in range(self.iterations): + start = perf_counter() + result = pickle.loads(self.pickle_data) + elapsed = perf_counter() - start + times.append(elapsed) + + # Verify correctness (first iteration only) + if len(times) == 1: + if result != self.obj: + raise ValueError("Unpickled object doesn't match original!") + + return { + 'mean': statistics.mean(times), + 'median': statistics.median(times), + 'stdev': statistics.stdev(times) if len(times) > 1 else 0.0, + 'min': min(times), + 'max': max(times), + } + + def benchmark_memory(self) -> int: + """Measure peak memory usage during unpickling.""" + tracemalloc.start() + + # Warmup + pickle.loads(self.pickle_data) + + # Actual measurement + gc.collect() + tracemalloc.reset_peak() + result = pickle.loads(self.pickle_data) + current, peak = tracemalloc.get_traced_memory() + + tracemalloc.stop() + + # Verify correctness + if result != self.obj: + raise ValueError("Unpickled object doesn't match original!") + + return peak + + def run_all(self) -> Dict[str, Any]: + """Run all benchmarks and return comprehensive results.""" + time_stats = self.benchmark_time() + peak_memory = self.benchmark_memory() + + return { + 'pickle_size_bytes': self.pickle_size, + 'pickle_size_mb': self.pickle_size / (1 << 20), + 'protocol': self.protocol, + 'time': time_stats, + 'memory_peak_bytes': peak_memory, + 'memory_peak_mb': peak_memory / (1 << 20), + 'iterations': self.iterations, + } + + +class AntagonisticBenchmark: + """Benchmark antagonistic/malicious pickles that demonstrate DoS protection. + + These pickles are designed to FAIL unpickling, but we measure peak memory + usage before the failure to demonstrate the memory protection. + """ + + def __init__(self, pickle_data: bytes, name: str): + self.pickle_data = pickle_data + self.name = name + + def measure_peak_memory(self, expect_success: bool = False) -> Dict[str, Any]: + """Measure peak memory when attempting to unpickle antagonistic data. + + Args: + expect_success: If True, test expects successful unpickling (e.g., sparse memo). + If False, test expects failure (e.g., truncated data). + """ + tracemalloc.start() + gc.collect() + tracemalloc.reset_peak() + + error_type = None + error_msg = None + succeeded = False + + try: + result = pickle.loads(self.pickle_data) + succeeded = True + if expect_success: + error_type = "Success (expected)" + else: + error_type = "WARNING: Expected failure but succeeded" + except (pickle.UnpicklingError, EOFError, ValueError, OverflowError) as e: + if expect_success: + error_type = f"UNEXPECTED FAILURE: {type(e).__name__}" + error_msg = str(e)[:100] + else: + # Expected failure for truncated data tests + error_type = type(e).__name__ + error_msg = str(e)[:100] + + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + return { + 'test_name': self.name, + 'peak_memory_bytes': peak, + 'peak_memory_mb': peak / (1 << 20), + 'error_type': error_type, + 'error_msg': error_msg, + 'pickle_size_bytes': len(self.pickle_data), + 'expected_outcome': 'success' if expect_success else 'failure', + 'succeeded': succeeded, + } + + +class AntagonisticTestSuite: + """Manage a suite of antagonistic (DoS protection) tests.""" + + # Default sizes in MB to claim (will provide only 1KB actual data) + DEFAULT_ANTAGONISTIC_SIZES_MB = [10, 50, 100, 500, 1000, 5000] + + def __init__(self, claimed_sizes_mb: List[int]): + self.claimed_sizes_mb = claimed_sizes_mb + + def _run_truncated_test( + self, + test_type: str, + generator_func, + claimed_bytes: int, + claimed_mb: int, + size_key: str, + all_results: Dict[str, Dict[str, Any]] + ) -> None: + """Run a single truncated data test and store results. + + Args: + test_type: Type identifier (e.g., 'binbytes8', 'binunicode8') + generator_func: Function to generate malicious pickle data + claimed_bytes: Size claimed in the pickle (bytes) + claimed_mb: Size claimed in the pickle (MB) + size_key: Result key for this size (e.g., '10MB') + all_results: Dictionary to store results in + """ + test_name = f"{test_type}_{size_key}_claim" + data = generator_func(claimed_bytes) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=False) + result['claimed_mb'] = claimed_mb + all_results[size_key][test_name] = result + + def run_all_tests(self) -> Dict[str, Dict[str, Any]]: + """Run comprehensive antagonistic test suite.""" + all_results = {} + + for claimed_mb in self.claimed_sizes_mb: + claimed_bytes = claimed_mb << 20 + size_key = f"{claimed_mb}MB" + all_results[size_key] = {} + + # Run truncated data tests (expect failure) + self._run_truncated_test('binbytes8', AntagonisticGenerator.truncated_binbytes8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('binunicode8', AntagonisticGenerator.truncated_binunicode8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('bytearray8', AntagonisticGenerator.truncated_bytearray8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('frame', AntagonisticGenerator.truncated_frame, + claimed_bytes, claimed_mb, size_key, all_results) + + # Test 5: Sparse memo (expect success - dict-based memo works!) + all_results["Sparse Memo (Success Expected)"] = {} + for test_name, (index, baseline_note) in SPARSE_MEMO_TESTS.items(): + data = AntagonisticGenerator.sparse_memo_attack(index) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=True) + result['claimed_mb'] = "N/A" + result['baseline_note'] = f"Without PR: {baseline_note}" + all_results["Sparse Memo (Success Expected)"][test_name] = result + + # Test 6: Multi-claim attack (expect failure) + test_name = "multi_claim_10x100MB" + data = AntagonisticGenerator.multi_claim_attack(10, 100 << 20) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=False) + result['claimed_mb'] = 1000 # 10 * 100MB + all_results["Multi-Claim (Failure Expected)"] = {test_name: result} + + return all_results + + +class TestSuite: + """Manage a suite of benchmark tests.""" + + def __init__(self, sizes: List[int], protocol: int = 5, iterations: int = 3): + self.sizes = sizes + self.protocol = protocol + self.iterations = iterations + self.results = {} + + def run_test(self, name: str, obj: Any) -> Dict[str, Any]: + """Run benchmark for a single test object.""" + bench = PickleBenchmark(obj, self.protocol, self.iterations) + results = bench.run_all() + results['test_name'] = name + results['object_type'] = type(obj).__name__ + return results + + def run_all_tests(self) -> Dict[str, Dict[str, Any]]: + """Run comprehensive test suite across all sizes and types.""" + all_results = {} + + for size in self.sizes: + size_key = f"{size / (1 << 20):.2f}MB" + all_results[size_key] = {} + + # Test 1: Large bytes object (BINBYTES8) + test_name = f"bytes_{size_key}" + obj = DataGenerator.large_bytes(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 2: Large ASCII string (BINUNICODE8) + test_name = f"string_ascii_{size_key}" + obj = DataGenerator.large_string_ascii(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 3: Large multibyte UTF-8 string + if size >= 3: + test_name = f"string_utf8_{size_key}" + obj = DataGenerator.large_string_multibyte(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 4: Large bytearray (BYTEARRAY8, protocol 5) + if self.protocol >= 5: + test_name = f"bytearray_{size_key}" + obj = DataGenerator.large_bytearray(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 5: List of large objects (repeated chunking) + if size >= MIN_READ_BUF_SIZE * 2: + test_name = f"list_large_items_{size_key}" + item_size = size // 5 + obj = DataGenerator.list_of_large_bytes(item_size, 5) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 6: Dict with large values + if size >= MIN_READ_BUF_SIZE * 2: + test_name = f"dict_large_values_{size_key}" + value_size = size // 3 + obj = DataGenerator.dict_with_large_values(value_size, 3) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 7: Nested structure + if size >= MIN_READ_BUF_SIZE: + test_name = f"nested_{size_key}" + obj = DataGenerator.nested_structure(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 8: Tuple (immutable) + if size >= 3: + test_name = f"tuple_{size_key}" + obj = DataGenerator.tuple_of_large_objects(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + return all_results + + +class Comparator: + """Compare benchmark results between current and baseline interpreters.""" + + @staticmethod + def _extract_json_from_output(output: str) -> Dict[str, Dict[str, Any]]: + """Extract JSON data from subprocess output. + + Skips any print statements before the JSON output and parses the JSON. + + Args: + output: Raw stdout from subprocess + + Returns: + Parsed JSON as dictionary + + Raises: + SystemExit: If JSON cannot be found or parsed + """ + output_lines = output.strip().split('\n') + json_start = -1 + for i, line in enumerate(output_lines): + if line.strip().startswith('{'): + json_start = i + break + + if json_start == -1: + print("Error: Could not find JSON output from baseline", file=sys.stderr) + sys.exit(1) + + json_output = '\n'.join(output_lines[json_start:]) + try: + return json.loads(json_output) + except json.JSONDecodeError as e: + print(f"Error: Could not parse baseline JSON output: {e}", file=sys.stderr) + sys.exit(1) + + @staticmethod + def run_baseline_benchmark(baseline_python: str, args: argparse.Namespace) -> Dict[str, Dict[str, Any]]: + """Run the benchmark using the baseline Python interpreter.""" + # Build command to run this script with baseline Python + cmd = [ + baseline_python, + __file__, + '--format', 'json', + '--protocol', str(args.protocol), + '--iterations', str(args.iterations), + ] + + if args.sizes is not None: + cmd.extend(['--sizes'] + [str(s) for s in args.sizes]) + + if args.antagonistic: + cmd.append('--antagonistic') + + print(f"\nRunning baseline benchmark with: {baseline_python}") + print(f"Command: {' '.join(cmd)}\n") + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=BASELINE_BENCHMARK_TIMEOUT_SECONDS, + ) + + if result.returncode != 0: + print(f"Error running baseline benchmark:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + + # Extract and parse JSON from output + return Comparator._extract_json_from_output(result.stdout) + + except subprocess.TimeoutExpired: + print("Error: Baseline benchmark timed out", file=sys.stderr) + sys.exit(1) + + @staticmethod + def calculate_change(baseline_value: float, current_value: float) -> float: + """Calculate percentage change from baseline to current.""" + if baseline_value == 0: + return 0.0 + return ((current_value - baseline_value) / baseline_value) * 100 + + @staticmethod + def format_comparison( + current_results: Dict[str, Dict[str, Any]], + baseline_results: Dict[str, Dict[str, Any]] + ) -> str: + """Format comparison results as readable text.""" + lines = [] + lines.append("=" * 100) + lines.append("Pickle Unpickling Benchmark Comparison") + lines.append("=" * 100) + lines.append("") + lines.append("Legend: Current vs Baseline | % Change (+ is slower/more memory, - is faster/less memory)") + lines.append("") + + # Sort size keys numerically + for size_key in sorted(current_results.keys(), key=_extract_size_mb): + if size_key not in baseline_results: + continue + + lines.append(f"\n{size_key} Comparison") + lines.append("-" * 100) + + current_tests = current_results[size_key] + baseline_tests = baseline_results[size_key] + + for test_name in sorted(current_tests.keys()): + if test_name not in baseline_tests: + continue + + curr = current_tests[test_name] + base = baseline_tests[test_name] + + time_change = Comparator.calculate_change( + base['time']['mean'], curr['time']['mean'] + ) + mem_change = Comparator.calculate_change( + base['memory_peak_mb'], curr['memory_peak_mb'] + ) + + lines.append(f"\n {curr['test_name']}") + lines.append(f" Time: {curr['time']['mean']*1000:6.2f}ms vs {base['time']['mean']*1000:6.2f}ms | " + f"{time_change:+6.1f}%") + lines.append(f" Memory: {curr['memory_peak_mb']:6.2f}MB vs {base['memory_peak_mb']:6.2f}MB | " + f"{mem_change:+6.1f}%") + + lines.append("\n" + "=" * 100) + lines.append("\nSummary:") + + # Calculate overall statistics + time_changes = [] + mem_changes = [] + + for size_key in current_results.keys(): + if size_key not in baseline_results: + continue + for test_name in current_results[size_key].keys(): + if test_name not in baseline_results[size_key]: + continue + curr = current_results[size_key][test_name] + base = baseline_results[size_key][test_name] + + time_changes.append(Comparator.calculate_change( + base['time']['mean'], curr['time']['mean'] + )) + mem_changes.append(Comparator.calculate_change( + base['memory_peak_mb'], curr['memory_peak_mb'] + )) + + if time_changes: + lines.append(f" Time change: mean={statistics.mean(time_changes):+.1f}%, " + f"median={statistics.median(time_changes):+.1f}%") + if mem_changes: + lines.append(f" Memory change: mean={statistics.mean(mem_changes):+.1f}%, " + f"median={statistics.median(mem_changes):+.1f}%") + + lines.append("=" * 100) + return "\n".join(lines) + + @staticmethod + def format_antagonistic_comparison( + current_results: Dict[str, Dict[str, Any]], + baseline_results: Dict[str, Dict[str, Any]] + ) -> str: + """Format antagonistic benchmark comparison results.""" + lines = [] + lines.append("=" * 100) + lines.append("Antagonistic Pickle Benchmark Comparison (Memory DoS Protection)") + lines.append("=" * 100) + lines.append("") + lines.append("Legend: Current vs Baseline | Memory Change (- is better, shows memory saved)") + lines.append("") + lines.append("This compares TWO types of DoS protection:") + lines.append(" 1. Truncated data → Baseline allocates full claimed size, Current uses chunked reading") + lines.append(" 2. Sparse memo → Baseline uses huge arrays, Current uses dict-based memo") + lines.append("") + + # Track statistics + truncated_memory_changes = [] + sparse_memory_changes = [] + + # Sort size keys numerically + for size_key in sorted(current_results.keys(), key=_extract_size_mb): + if size_key not in baseline_results: + continue + + lines.append(f"\n{size_key} Comparison") + lines.append("-" * 100) + + current_tests = current_results[size_key] + baseline_tests = baseline_results[size_key] + + for test_name in sorted(current_tests.keys()): + if test_name not in baseline_tests: + continue + + curr = current_tests[test_name] + base = baseline_tests[test_name] + + curr_peak_mb = curr['peak_memory_mb'] + base_peak_mb = base['peak_memory_mb'] + expected_outcome = curr.get('expected_outcome', 'failure') + + mem_change = Comparator.calculate_change(base_peak_mb, curr_peak_mb) + mem_saved_mb = base_peak_mb - curr_peak_mb + + lines.append(f"\n {curr['test_name']}") + lines.append(f" Memory: {curr_peak_mb:6.2f}MB vs {base_peak_mb:6.2f}MB | " + f"{mem_change:+6.1f}% ({mem_saved_mb:+.2f}MB saved)") + + # Track based on test type + if expected_outcome == 'success': + sparse_memory_changes.append(mem_change) + if curr.get('baseline_note'): + lines.append(f" Note: {curr['baseline_note']}") + else: + truncated_memory_changes.append(mem_change) + claimed_mb = curr.get('claimed_mb', 'N/A') + if claimed_mb != 'N/A': + lines.append(f" Claimed: {claimed_mb:,}MB") + + # Show status + curr_status = curr.get('error_type', 'Unknown') + base_status = base.get('error_type', 'Unknown') + if curr_status != base_status: + lines.append(f" Status: {curr_status} (baseline: {base_status})") + else: + lines.append(f" Status: {curr_status}") + + lines.append("\n" + "=" * 100) + lines.append("\nSummary:") + lines.append("") + + if truncated_memory_changes: + lines.append(" Truncated Data Protection (chunked reading):") + lines.append(f" Mean memory change: {statistics.mean(truncated_memory_changes):+.1f}%") + lines.append(f" Median memory change: {statistics.median(truncated_memory_changes):+.1f}%") + avg_change = statistics.mean(truncated_memory_changes) + if avg_change < -50: + lines.append(f" Result: ✓ Dramatic memory reduction ({avg_change:.1f}%) - DoS protection working!") + elif avg_change < 0: + lines.append(f" Result: ✓ Memory reduced ({avg_change:.1f}%)") + else: + lines.append(f" Result: ⚠ Memory increased ({avg_change:.1f}%) - unexpected!") + lines.append("") + + if sparse_memory_changes: + lines.append(" Sparse Memo Protection (dict-based memo):") + lines.append(f" Mean memory change: {statistics.mean(sparse_memory_changes):+.1f}%") + lines.append(f" Median memory change: {statistics.median(sparse_memory_changes):+.1f}%") + avg_change = statistics.mean(sparse_memory_changes) + if avg_change < -50: + lines.append(f" Result: ✓ Dramatic memory reduction ({avg_change:.1f}%) - Dict optimization working!") + elif avg_change < 0: + lines.append(f" Result: ✓ Memory reduced ({avg_change:.1f}%)") + else: + lines.append(f" Result: ⚠ Memory increased ({avg_change:.1f}%) - unexpected!") + + lines.append("") + lines.append("=" * 100) + return "\n".join(lines) + + +class Reporter: + """Format and display benchmark results.""" + + @staticmethod + def format_text(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as readable text.""" + lines = [] + lines.append("=" * 80) + lines.append("Pickle Unpickling Benchmark Results") + lines.append("=" * 80) + lines.append("") + + for size_key, tests in results.items(): + lines.append(f"\n{size_key} Test Results") + lines.append("-" * 80) + + for test_name, data in tests.items(): + lines.append(f"\n Test: {data['test_name']}") + lines.append(f" Type: {data['object_type']}") + lines.append(f" Pickle size: {data['pickle_size_mb']:.2f} MB") + lines.append(f" Time (mean): {data['time']['mean']*1000:.2f} ms") + lines.append(f" Time (stdev): {data['time']['stdev']*1000:.2f} ms") + lines.append(f" Peak memory: {data['memory_peak_mb']:.2f} MB") + lines.append(f" Protocol: {data['protocol']}") + + lines.append("\n" + "=" * 80) + return "\n".join(lines) + + @staticmethod + def format_markdown(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as markdown table.""" + lines = [] + lines.append("# Pickle Unpickling Benchmark Results\n") + + for size_key, tests in results.items(): + lines.append(f"## {size_key}\n") + lines.append("| Test | Type | Pickle Size (MB) | Time (ms) | Stdev (ms) | Peak Memory (MB) |") + lines.append("|------|------|------------------|-----------|------------|------------------|") + + for test_name, data in tests.items(): + lines.append( + f"| {data['test_name']} | " + f"{data['object_type']} | " + f"{data['pickle_size_mb']:.2f} | " + f"{data['time']['mean']*1000:.2f} | " + f"{data['time']['stdev']*1000:.2f} | " + f"{data['memory_peak_mb']:.2f} |" + ) + lines.append("") + + return "\n".join(lines) + + @staticmethod + def format_json(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as JSON.""" + import json + return json.dumps(results, indent=2) + + @staticmethod + def format_antagonistic(results: Dict[str, Dict[str, Any]]) -> str: + """Format antagonistic benchmark results.""" + lines = [] + lines.append("=" * 100) + lines.append("Antagonistic Pickle Benchmark (Memory DoS Protection Test)") + lines.append("=" * 100) + lines.append("") + lines.append("This benchmark tests TWO types of DoS protection:") + lines.append(" 1. Truncated data attacks → Expect FAILURE with minimal memory before failure") + lines.append(" 2. Sparse memo attacks → Expect SUCCESS with dict-based memo (vs huge array)") + lines.append("") + + # Sort size keys numerically + for size_key in sorted(results.keys(), key=_extract_size_mb): + tests = results[size_key] + + # Determine test type from first test + if tests: + first_test = next(iter(tests.values())) + expected_outcome = first_test.get('expected_outcome', 'failure') + claimed_mb = first_test.get('claimed_mb', 'N/A') + + # Header varies by test type + if "Sparse Memo" in size_key: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + elif "Multi-Claim" in size_key: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + elif claimed_mb != 'N/A': + lines.append(f"\n{size_key} Claimed (actual: 1KB) - Expect Failure") + lines.append("-" * 100) + else: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + + for test_name, data in tests.items(): + peak_mb = data['peak_memory_mb'] + claimed = data.get('claimed_mb', 'N/A') + expected_outcome = data.get('expected_outcome', 'failure') + succeeded = data.get('succeeded', False) + baseline_note = data.get('baseline_note', '') + + lines.append(f" {data['test_name']}") + + # Format output based on test type + if expected_outcome == 'success': + # Sparse memo test - show success with dict + status_icon = "✓" if succeeded else "✗" + lines.append(f" Peak memory: {peak_mb:8.2f} MB {status_icon}") + lines.append(f" Status: {data['error_type']}") + if baseline_note: + lines.append(f" {baseline_note}") + else: + # Truncated data test - show savings before failure + if claimed != 'N/A': + saved_mb = claimed - peak_mb + savings_pct = (saved_mb / claimed * 100) if claimed > 0 else 0 + lines.append(f" Peak memory: {peak_mb:8.2f} MB (claimed: {claimed:,} MB, saved: {saved_mb:.2f} MB, {savings_pct:.1f}%)") + else: + lines.append(f" Peak memory: {peak_mb:8.2f} MB") + lines.append(f" Status: {data['error_type']}") + + lines.append("\n" + "=" * 100) + + # Calculate statistics by test type + truncated_claimed = 0 + truncated_peak = 0 + truncated_count = 0 + + sparse_peak_total = 0 + sparse_count = 0 + + for size_key, tests in results.items(): + for test_name, data in tests.items(): + expected_outcome = data.get('expected_outcome', 'failure') + + if expected_outcome == 'failure': + # Truncated data test + claimed = data.get('claimed_mb', 0) + if claimed != 'N/A' and claimed > 0: + truncated_claimed += claimed + truncated_peak += data['peak_memory_mb'] + truncated_count += 1 + else: + # Sparse memo test + sparse_peak_total += data['peak_memory_mb'] + sparse_count += 1 + + lines.append("\nSummary:") + lines.append("") + + if truncated_count > 0: + avg_claimed = truncated_claimed / truncated_count + avg_peak = truncated_peak / truncated_count + avg_saved = avg_claimed - avg_peak + avg_savings_pct = (avg_saved / avg_claimed * 100) if avg_claimed > 0 else 0 + + lines.append(" Truncated Data Protection (chunked reading):") + lines.append(f" Average claimed: {avg_claimed:,.1f} MB") + lines.append(f" Average peak: {avg_peak:,.2f} MB") + lines.append(f" Average saved: {avg_saved:,.2f} MB ({avg_savings_pct:.1f}% reduction)") + lines.append(f" Status: ✓ Fails fast with minimal memory") + lines.append("") + + if sparse_count > 0: + avg_sparse_peak = sparse_peak_total / sparse_count + lines.append(" Sparse Memo Protection (dict-based memo):") + lines.append(f" Average peak: {avg_sparse_peak:,.2f} MB") + lines.append(f" Status: ✓ Succeeds with dict (vs GB-sized arrays without PR)") + lines.append(f" Note: Compare with --baseline to see actual memory savings") + + lines.append("") + lines.append("=" * 100) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark pickle unpickling performance for large objects" + ) + parser.add_argument( + '--sizes', + type=int, + nargs='+', + default=None, + metavar='MiB', + help=f'Object sizes to test in MiB (default: {DEFAULT_SIZES_MIB})' + ) + parser.add_argument( + '--protocol', + type=int, + default=5, + choices=[0, 1, 2, 3, 4, 5], + help='Pickle protocol version (default: 5)' + ) + parser.add_argument( + '--iterations', + type=int, + default=3, + help='Number of benchmark iterations (default: 3)' + ) + parser.add_argument( + '--format', + choices=['text', 'markdown', 'json'], + default='text', + help='Output format (default: text)' + ) + parser.add_argument( + '--baseline', + type=str, + metavar='PYTHON', + help='Path to baseline Python interpreter for comparison (e.g., ../main-build/python)' + ) + parser.add_argument( + '--antagonistic', + action='store_true', + help='Run antagonistic/malicious pickle tests (DoS protection benchmark)' + ) + + args = parser.parse_args() + + # Handle antagonistic mode + if args.antagonistic: + # Antagonistic mode uses claimed sizes in MB, not actual data sizes + if args.sizes is None: + claimed_sizes_mb = AntagonisticTestSuite.DEFAULT_ANTAGONISTIC_SIZES_MB + else: + claimed_sizes_mb = args.sizes + + print(f"Running ANTAGONISTIC pickle benchmark (DoS protection test)...") + print(f"Claimed sizes: {claimed_sizes_mb} MiB (actual data: 1KB each)") + print(f"NOTE: These pickles will FAIL to unpickle (expected)") + print() + + # Run antagonistic benchmark suite + suite = AntagonisticTestSuite(claimed_sizes_mb) + results = suite.run_all_tests() + + # Format and display results + if args.baseline: + # Verify baseline Python exists + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"Error: Baseline Python not found: {args.baseline}", file=sys.stderr) + return 1 + + # Run baseline benchmark + baseline_results = Comparator.run_baseline_benchmark(args.baseline, args) + + # Show comparison + comparison_output = Comparator.format_antagonistic_comparison(results, baseline_results) + print(comparison_output) + else: + # Format and display results + output = _format_output(results, args.format, is_antagonistic=True) + print(output) + + else: + # Normal mode: legitimate pickle benchmarks + # Convert sizes from MiB to bytes + if args.sizes is None: + sizes_bytes = DEFAULT_SIZES + else: + sizes_bytes = [size * (1 << 20) for size in args.sizes] + + print(f"Running pickle benchmark with protocol {args.protocol}...") + print(f"Test sizes: {[f'{s/(1<<20):.2f}MiB' for s in sizes_bytes]}") + print(f"Iterations per test: {args.iterations}") + print() + + # Run benchmark suite + suite = TestSuite(sizes_bytes, args.protocol, args.iterations) + results = suite.run_all_tests() + + # If baseline comparison requested, run baseline and compare + if args.baseline: + # Verify baseline Python exists + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"Error: Baseline Python not found: {args.baseline}", file=sys.stderr) + return 1 + + # Run baseline benchmark + baseline_results = Comparator.run_baseline_benchmark(args.baseline, args) + + # Show comparison + comparison_output = Comparator.format_comparison(results, baseline_results) + print(comparison_output) + + else: + # Format and display results + output = _format_output(results, args.format, is_antagonistic=False) + print(output) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From dcac498e501db3983ce22851c88f88561ae46351 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 5 Dec 2025 19:36:28 +0000 Subject: [PATCH 411/417] gh-142318: Fix typing `'q'` at interactive help screen exiting Tachyon (#142319) --- Lib/profiling/sampling/live_collector/collector.py | 4 +++- .../test_live_collector_interaction.py | 13 +++++++++++++ .../2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index 4b69275a2f0..7adbf1bbe7f 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -861,10 +861,12 @@ def _handle_input(self): # Handle help toggle keys if ch == ord("h") or ch == ord("H") or ch == ord("?"): self.show_help = not self.show_help + return # If showing help, any other key closes it - elif self.show_help and ch != -1: + if self.show_help and ch != -1: self.show_help = False + return # Handle regular commands if ch == ord("q") or ch == ord("Q"): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py index 388f462cf21..a5870366552 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py @@ -173,6 +173,19 @@ def test_help_with_question_mark(self): self.assertTrue(self.collector.show_help) + def test_help_dismiss_with_q_does_not_quit(self): + """Test that pressing 'q' while help is shown only closes help, not quit""" + self.assertFalse(self.collector.show_help) + self.display.simulate_input(ord("h")) + self.collector._handle_input() + self.assertTrue(self.collector.show_help) + + self.display.simulate_input(ord("q")) + self.collector._handle_input() + + self.assertFalse(self.collector.show_help) + self.assertTrue(self.collector.running) + def test_filter_clear(self): """Test clearing filter.""" self.collector.filter_pattern = "test" diff --git a/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst b/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst new file mode 100644 index 00000000000..8710ebfb1a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst @@ -0,0 +1,2 @@ +Fix typing ``'q'`` at the help of the interactive tachyon profiler exiting +the profiler. From 58e1c7a16f0926b1047c336eeed2849d5fff7c70 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 5 Dec 2025 13:35:50 -0800 Subject: [PATCH 412/417] Introduce `build-python` and `build-host` subcommands for `Tools/wasm/wasi` (GH-142266) It should make it easier when you need to rebuild just the e.g. host Python, but it requires ./configure to run. Co-authored-by: Emma Smith --- Tools/wasm/wasi/__main__.py | 56 ++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index d95cc99c8ea..7e33ff4b477 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -386,18 +386,6 @@ def make_wasi_python(context, working_dir): ) -def build_all(context): - """Build everything.""" - steps = [ - configure_build_python, - make_build_python, - configure_wasi_python, - make_wasi_python, - ] - for step in steps: - step(context) - - def clean_contents(context): """Delete all files created by this script.""" if CROSS_BUILD_DIR.exists(): @@ -409,6 +397,16 @@ def clean_contents(context): log("🧹", f"Deleting generated {LOCAL_SETUP} ...") +def build_steps(*steps): + """Construct a command from other steps.""" + + def builder(context): + for step in steps: + step(context) + + return builder + + def main(): default_host_triple = "wasm32-wasip1" default_wasi_sdk = find_wasi_sdk() @@ -438,6 +436,9 @@ def main(): make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) + build_python = subcommands.add_parser( + "build-python", help="Build the build Python" + ) configure_host = subcommands.add_parser( "configure-host", help="Run `configure` for the " @@ -448,6 +449,9 @@ def main(): make_host = subcommands.add_parser( "make-host", help="Run `make` for the host/WASI" ) + build_host = subcommands.add_parser( + "build-host", help="Build the host/WASI Python" + ) subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) @@ -455,8 +459,10 @@ def main(): build, configure_build, make_build, + build_python, configure_host, make_host, + build_host, ): subcommand.add_argument( "--quiet", @@ -471,7 +477,12 @@ def main(): default=default_logdir, help=f"Directory to store log files; defaults to {default_logdir}", ) - for subcommand in configure_build, configure_host: + for subcommand in ( + configure_build, + configure_host, + build_python, + build_host, + ): subcommand.add_argument( "--clean", action="store_true", @@ -479,11 +490,17 @@ def main(): dest="clean", help="Delete any relevant directories before building", ) - for subcommand in build, configure_build, configure_host: + for subcommand in ( + build, + configure_build, + configure_host, + build_python, + build_host, + ): subcommand.add_argument( "args", nargs="*", help="Extra arguments to pass to `configure`" ) - for subcommand in build, configure_host: + for subcommand in build, configure_host, build_host: subcommand.add_argument( "--wasi-sdk", type=pathlib.Path, @@ -499,7 +516,7 @@ def main(): help="Command template for running the WASI host; defaults to " f"`{default_host_runner}`", ) - for subcommand in build, configure_host, make_host: + for subcommand in build, configure_host, make_host, build_host: subcommand.add_argument( "--host-triple", action="store", @@ -511,12 +528,17 @@ def main(): context = parser.parse_args() context.init_dir = pathlib.Path().absolute() + build_build_python = build_steps(configure_build_python, make_build_python) + build_wasi_python = build_steps(configure_wasi_python, make_wasi_python) + dispatch = { "configure-build-python": configure_build_python, "make-build-python": make_build_python, + "build-python": build_build_python, "configure-host": configure_wasi_python, "make-host": make_wasi_python, - "build": build_all, + "build-host": build_wasi_python, + "build": build_steps(build_build_python, build_wasi_python), "clean": clean_contents, } dispatch[context.subcommand](context) From d49e6f38a7a0ca666df2c81329291291f0389682 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 5 Dec 2025 14:31:30 -0800 Subject: [PATCH 413/417] Extract data from `Tools/wasm/wasi` that varies between Python versions into a config file (GH-142273) This should allow for easier backporting of code. --- Tools/wasm/wasi/__main__.py | 18 ++++++++++++------ Tools/wasm/wasi/config.toml | 6 ++++++ 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 Tools/wasm/wasi/config.toml diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 7e33ff4b477..3d9a2472d4d 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -5,6 +5,8 @@ import functools import os +import tomllib + try: from os import process_cpu_count as cpu_count except ImportError: @@ -18,6 +20,7 @@ HERE = pathlib.Path(__file__).parent +# Path is: cpython/Tools/wasm/wasi CHECKOUT = HERE.parent.parent.parent assert (CHECKOUT / "configure").is_file(), ( "Please update the location of the file" @@ -213,9 +216,10 @@ def make_build_python(context, working_dir): log("🎉", f"{binary} {version}") -def find_wasi_sdk(): +def find_wasi_sdk(config): """Find the path to the WASI SDK.""" wasi_sdk_path = None + wasi_sdk_version = config["targets"]["wasi-sdk"] if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) @@ -229,7 +233,7 @@ def find_wasi_sdk(): # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. potential_sdks = [ path - for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") if path.is_dir() ] if len(potential_sdks) == 1: @@ -245,12 +249,12 @@ def find_wasi_sdk(): found_version = version_details.splitlines()[0] # Make sure there's a trailing dot to avoid false positives if somehow the # supported version is a prefix of the found version (e.g. `25` and `2567`). - if not found_version.startswith(f"{WASI_SDK_VERSION}."): + if not found_version.startswith(f"{wasi_sdk_version}."): major_version = found_version.partition(".")[0] log( "⚠️", f" Found WASI SDK {major_version}, " - f"but WASI SDK {WASI_SDK_VERSION} is the supported version", + f"but WASI SDK {wasi_sdk_version} is the supported version", ) return wasi_sdk_path @@ -408,8 +412,10 @@ def builder(context): def main(): - default_host_triple = "wasm32-wasip1" - default_wasi_sdk = find_wasi_sdk() + with (HERE / "config.toml").open("rb") as file: + config = tomllib.load(file) + default_wasi_sdk = find_wasi_sdk(config) + default_host_triple = config["targets"]["host-triple"] default_host_runner = ( f"{WASMTIME_HOST_RUNNER_VAR} run " # For setting PYTHONPATH to the sysconfig data directory. diff --git a/Tools/wasm/wasi/config.toml b/Tools/wasm/wasi/config.toml new file mode 100644 index 00000000000..7ca2f76f56d --- /dev/null +++ b/Tools/wasm/wasi/config.toml @@ -0,0 +1,6 @@ +# Any data that can vary between Python versions is to be kept in this file. +# This allows for blanket copying of the WASI build code between supported +# Python versions. +[targets] +wasi-sdk = 29 +host-triple = "wasm32-wasip1" From eba449a1989265a923174142dd67dee074f90967 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 5 Dec 2025 15:27:16 -0800 Subject: [PATCH 414/417] GH-142234: Allow `--enable-wasm-dynamic-linking` under WASI (GH-142235) While CPython doesn't support `--enable-wasm-dynamic-linking`, external tools like componentize-py do and they have to patch around it. Since the flag is off by default, allowing the flag so external users can add/inject dynamic linking support seems acceptable. --- .../Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst | 3 +++ configure | 5 +++-- configure.ac | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst diff --git a/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst b/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst new file mode 100644 index 00000000000..a586512fc0a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst @@ -0,0 +1,3 @@ +Allow ``--enable-wasm-dynamic-linking`` for WASI. While CPython doesn't +directly support it so external/downstream users do not have to patch in +support for the flag. diff --git a/configure b/configure index 1561f7f4134..7561fb9c7ad 100755 --- a/configure +++ b/configure @@ -1824,7 +1824,8 @@ Optional Features: no) --enable-wasm-dynamic-linking Enable dynamic linking support for WebAssembly - (default is no) + (default is no); WASI requires an external dynamic + loader to handle imports --enable-wasm-pthreads Enable pthread emulation for WebAssembly (default is no) --enable-shared enable building a shared Python library (default is @@ -7415,7 +7416,7 @@ then : Emscripten) : ;; #( WASI) : - as_fn_error $? "WASI dynamic linking is not implemented yet." "$LINENO" 5 ;; #( + ;; #( *) : as_fn_error $? "--enable-wasm-dynamic-linking only applies to Emscripten and WASI" "$LINENO" 5 ;; diff --git a/configure.ac b/configure.ac index f2a7319d22d..fa24bc78a26 100644 --- a/configure.ac +++ b/configure.ac @@ -1323,11 +1323,11 @@ dnl See https://emscripten.org/docs/compiling/Dynamic-Linking.html AC_MSG_CHECKING([for --enable-wasm-dynamic-linking]) AC_ARG_ENABLE([wasm-dynamic-linking], [AS_HELP_STRING([--enable-wasm-dynamic-linking], - [Enable dynamic linking support for WebAssembly (default is no)])], + [Enable dynamic linking support for WebAssembly (default is no); WASI requires an external dynamic loader to handle imports])], [ AS_CASE([$ac_sys_system], [Emscripten], [], - [WASI], [AC_MSG_ERROR([WASI dynamic linking is not implemented yet.])], + [WASI], [], [AC_MSG_ERROR([--enable-wasm-dynamic-linking only applies to Emscripten and WASI])] ) ], [ From d1194439363a6896069c5c3cf66e95a14f79bc00 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 6 Dec 2025 12:27:31 +0100 Subject: [PATCH 415/417] Remove unused imports (#142320) --- Lib/cProfile.py | 1 - Lib/profiling/sampling/heatmap_collector.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 4af82f2cb8c..cc6255f61ae 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -9,6 +9,5 @@ __all__ = ["run", "runctx", "Profile"] if __name__ == "__main__": - import sys from profiling.tracing.__main__ import main main() diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index eb51ce33b28..eb128aba9b1 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -11,7 +11,7 @@ import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Tuple, Optional, Any +from typing import Dict, List, Tuple from ._css_utils import get_combined_css from .stack_collector import StackTraceCollector From 61823a5382e8c0c0292e90a46ae3e1859b7f278b Mon Sep 17 00:00:00 2001 From: "Y. Z. Chen" <754097987@qq.com> Date: Sat, 6 Dec 2025 21:05:20 +0800 Subject: [PATCH 416/417] Docs: fix RFC index reference for TLS 1.3 (#142262) --- Lib/ssl.py | 2 +- Lib/test/test_ssl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 7ad7969a821..67a2990b281 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -185,7 +185,7 @@ class _TLSContentType: class _TLSAlertType: """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 09de32f8371..ebdf5455163 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -5613,7 +5613,7 @@ def test_tlsalerttype(self): class Checked_TLSAlertType(enum.IntEnum): """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 From 5be3405e4e94e494f3f2c4507d8c32c2c04bb2ee Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 6 Dec 2025 07:12:21 -0800 Subject: [PATCH 417/417] GH-75949: Fix argparse dropping '|' in mutually exclusive groups on line wrap (#142312) --- Lib/argparse.py | 33 ++++++++++++++++--- Lib/test/test_argparse.py | 19 +++++++++++ ...5-12-05-16-39-17.gh-issue-75949.pHxW98.rst | 1 + 3 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 10393b6a02b..07d7d77e884 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -353,8 +353,14 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + # keep optionals and positionals together to preserve + # mutually exclusive group formatting (gh-75949) + all_actions = optionals + positionals + parts, pos_start = self._get_actions_usage_parts_with_split( + all_actions, groups, len(optionals) + ) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -418,6 +424,17 @@ def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): + parts, _ = self._get_actions_usage_parts_with_split(actions, groups) + return parts + + def _get_actions_usage_parts_with_split(self, actions, groups, opt_count=None): + """Get usage parts with split index for optionals/positionals. + + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. When opt_count is None, pos_start is None. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -513,8 +530,16 @@ def _get_actions_usage_parts(self, actions, groups): for i in range(start + group_size, end): parts[i] = None - # return the usage parts - return [item for item in parts if item is not None] + # if opt_count is provided, calculate where positionals start in + # the final parts list (for wrapping onto separate lines). + # Count before filtering None entries since indices shift after. + if opt_count is not None: + pos_start = sum(1 for p in parts[:opt_count] if p is not None) + else: + pos_start = None + + # return the usage parts and split point (gh-75949) + return [item for item in parts if item is not None], pos_start def _format_text(self, text): if '%(prog)' in text: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ef90d4bcbb2..dff7ba750fa 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4966,6 +4966,25 @@ def test_long_mutex_groups_wrap(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | + positional] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" diff --git a/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst b/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst new file mode 100644 index 00000000000..5ca3fc05b98 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst @@ -0,0 +1 @@ +Fix :mod:`argparse` to preserve ``|`` separators in mutually exclusive groups when the usage line wraps due to length.