Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-12-07 13:50:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-141794: Limit size of generated machine code. (GH-142228)

Browse source 
* Factor out bodies of the largest uops, to reduce jit code size.
* Factor out common assert, also reducing jit code size.
* Limit size of jitted code for a single executor to 1MB.
This commit is contained in:
Mark Shannon 2025-12-03 17:43:35 +00:00 • committed by GitHub
parent aea5531583
commit 62423c9c36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1406 additions and 1731 deletions
Download patch file Download diff file Expand all files Collapse all files

58
Include/internal/pycore_ceval.h
View file

@ -408,6 +408,64 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame
PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR;
/* Helper functions for large uops */
PyAPI_FUNC(PyObject *)
_Py_VectorCall_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args,
_PyStackRef kwnames);
PyAPI_FUNC(PyObject *)
_Py_BuiltinCallFast_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args);
PyAPI_FUNC(PyObject *)
_Py_BuiltinCallFastWithKeywords_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args);
PyAPI_FUNC(PyObject *)
_PyCallMethodDescriptorFast_StackRefSteal(
_PyStackRef callable,
PyMethodDef *meth,
PyObject *self,
_PyStackRef *arguments,
int total_args);
PyAPI_FUNC(PyObject *)
_PyCallMethodDescriptorFastWithKeywords_StackRefSteal(
_PyStackRef callable,
PyMethodDef *meth,
PyObject *self,
_PyStackRef *arguments,
int total_args);
PyAPI_FUNC(PyObject *)
_Py_CallBuiltinClass_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args);
PyAPI_FUNC(PyObject *)
_Py_BuildString_StackRefSteal(
_PyStackRef *arguments,
int total_args);
PyAPI_FUNC(PyObject *)
_Py_BuildMap_StackRefSteal(
_PyStackRef *arguments,
int half_args);
PyAPI_FUNC(void)
_Py_assert_within_stack_bounds(
_PyInterpreterFrame *frame, _PyStackRef *stack_pointer,
const char *filename, int lineno);
#ifdef __cplusplus
}
#endif

3
Include/internal/pycore_jit.h
View file

@ -13,6 +13,9 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
/* To be able to reason about code layout and branches, keep code size below 1 MB */
#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1)
#ifdef _Py_JIT
typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate);

5
Include/internal/pycore_uop.h
View file

@ -36,7 +36,12 @@ typedef struct _PyUOpInstruction{
} _PyUOpInstruction;
// This is the length of the trace we translate initially.
#ifdef Py_DEBUG
// With asserts, the stencils are a lot larger
#define UOP_MAX_TRACE_LENGTH 1000
#else
#define UOP_MAX_TRACE_LENGTH 3000
#endif
#define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction))
/* Bloom filter with m = 256

54
Lib/test/test_generated_cases.py
View file

@ -165,7 +165,7 @@ def test_inst_one_pop(self):
value = stack_pointer[-1];
SPAM(value);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -190,7 +190,7 @@ def test_inst_one_push(self):
res = SPAM();
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -247,7 +247,7 @@ def test_binary_op(self):
res = SPAM(left, right);
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -366,14 +366,14 @@ def test_sync_sp(self):
_PyStackRef res;
arg = stack_pointer[-1];
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
escaping_call();
stack_pointer = _PyFrame_GetStackPointer(frame);
res = Py_None;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
@ -489,7 +489,7 @@ def test_error_if_pop(self):
res = 0;
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -523,7 +523,7 @@ def test_error_if_pop_with_result(self):
}
stack_pointer[-2] = res;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -553,7 +553,7 @@ def test_cache_effect(self):
uint32_t extra = read_u32(&this_instr[2].cache);
(void)extra;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -640,7 +640,7 @@ def test_macro_instruction(self):
}
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
@ -688,7 +688,7 @@ def test_macro_instruction(self):
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer[-3] = res;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -827,7 +827,7 @@ def test_array_input(self):
below = stack_pointer[-2 - oparg*2];
SPAM(values, oparg);
stack_pointer += -2 - oparg*2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -860,7 +860,7 @@ def test_array_output(self):
stack_pointer[-2] = below;
stack_pointer[-1 + oparg*3] = above;
stack_pointer += oparg*3;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -889,7 +889,7 @@ def test_array_input_output(self):
above = 0;
stack_pointer[0] = above;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -918,11 +918,11 @@ def test_array_error_if(self):
extra = stack_pointer[-1 - oparg];
if (oparg == 0) {
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_LABEL(error);
}
stack_pointer += -1 - oparg;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -960,7 +960,7 @@ def test_macro_push_push(self):
stack_pointer[0] = val1;
stack_pointer[1] = val2;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -1263,13 +1263,13 @@ def test_flush(self):
stack_pointer[0] = a;
stack_pointer[1] = b;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
// SECOND
{
USE(a, b);
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -1325,7 +1325,7 @@ def test_pop_on_error_peeks(self):
}
}
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -1368,14 +1368,14 @@ def test_push_then_error(self):
stack_pointer[0] = a;
stack_pointer[1] = b;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = a;
stack_pointer[1] = b;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -1661,7 +1661,7 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self):
stack_pointer[0] = out1;
stack_pointer[1] = out2;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
"""
@ -1881,7 +1881,7 @@ def test_reassigning_dead_inputs(self):
stack_pointer = _PyFrame_GetStackPointer(frame);
in = temp;
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(in);
stack_pointer = _PyFrame_GetStackPointer(frame);
@ -2116,7 +2116,7 @@ def test_validate_uop_unused_input(self):
output = """
case OP: {
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
"""
@ -2133,7 +2133,7 @@ def test_validate_uop_unused_input(self):
output = """
case OP: {
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
"""
@ -2155,7 +2155,7 @@ def test_validate_uop_unused_output(self):
foo = NULL;
stack_pointer[0] = foo;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
"""
@ -2173,7 +2173,7 @@ def test_validate_uop_unused_output(self):
output = """
case OP: {
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
"""

179
Python/bytecodes.c
View file

@ -1976,14 +1976,8 @@ dummy_func(
}
inst(BUILD_STRING, (pieces[oparg] -- str)) {
STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o);
if (CONVERSION_FAILED(pieces_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o);
DECREF_INPUTS();
PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg);
DEAD(pieces);
ERROR_IF(str_o == NULL);
str = PyStackRef_FromPyObjectSteal(str_o);
}
@ -2098,17 +2092,9 @@ dummy_func(
}
inst(BUILD_MAP, (values[oparg*2] -- map)) {
STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o);
if (CONVERSION_FAILED(values_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *map_o = _PyDict_FromItems(
values_o, 2,
values_o+1, 2,
oparg);
STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
DECREF_INPUTS();
PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg);
DEAD(values);
ERROR_IF(map_o == NULL);
map = PyStackRef_FromPyObjectStealMortal(map_o);
}
@ -3891,27 +3877,20 @@ dummy_func(
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
_PyStackRef *arguments = args;
if (!PyStackRef_IsNull(self_or_null)) {
arguments--;
total_args++;
}
/* Callable is not a normal Python function */
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *res_o = PyObject_Vectorcall(
callable_o, args_o,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
DECREF_INPUTS();
PyObject *res_o = _Py_VectorCall_StackRefSteal(
callable,
arguments,
total_args,
PyStackRef_NULL);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}
@ -4186,14 +4165,13 @@ dummy_func(
}
DEOPT_IF(tp->tp_vectorcall == NULL);
STAT_INC(CALL, hit);
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
DECREF_INPUTS();
PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal(
callable,
arguments,
total_args);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}
@ -4241,31 +4219,24 @@ dummy_func(
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_FASTCALL functions, without keywords */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
_PyStackRef *arguments = args;
if (!PyStackRef_IsNull(self_or_null)) {
arguments--;
total_args++;
}
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(!PyCFunction_CheckExact(callable_o));
DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
/* res = func(self, args, nargs) */
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *res_o = _PyCFunctionFast_CAST(cfunc)(
PyCFunction_GET_SELF(callable_o),
args_o,
total_args);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
DECREF_INPUTS();
PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal(
callable,
arguments,
total_args
);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}
@ -4278,30 +4249,20 @@ dummy_func(
op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
_PyStackRef *arguments = args;
if (!PyStackRef_IsNull(self_or_null)) {
arguments--;
total_args++;
}
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
DEOPT_IF(!PyCFunction_CheckExact(callable_o));
DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS));
STAT_INC(CALL, hit);
/* res = func(self, arguments, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
_PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o));
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
DECREF_INPUTS();
PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}
@ -4468,19 +4429,16 @@ dummy_func(
assert(self != NULL);
EXIT_IF(!Py_IS_TYPE(self, d_type));
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyCFunctionFastWithKeywords cfunc =
_PyCFunctionFastWithKeywords_CAST(meth->ml_meth);
PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
DECREF_INPUTS();
PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal(
callable,
meth,
self,
arguments,
total_args
);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}
@ -4548,18 +4506,16 @@ dummy_func(
assert(self != NULL);
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
STAT_INC(CALL, hit);
int nargs = total_args - 1;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth);
PyObject *res_o = cfunc(self, (args_o + 1), nargs);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
DECREF_INPUTS();
PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal(
callable,
meth,
self,
arguments,
total_args
);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}
@ -4792,30 +4748,21 @@ dummy_func(
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
int total_args = oparg;
_PyStackRef *arguments = args;
if (!PyStackRef_IsNull(self_or_null)) {
arguments--;
total_args++;
}
/* Callable is not a normal Python function */
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
DECREF_INPUTS();
ERROR_IF(true);
}
PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o);
PyObject *res_o = PyObject_Vectorcall(
callable_o, args_o,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames_o);
PyStackRef_CLOSE(kwnames);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
DECREF_INPUTS();
PyObject *res_o = _Py_VectorCall_StackRefSteal(
callable,
arguments,
total_args,
kwnames);
DEAD(kwnames);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
}

284
Python/ceval.c
View file

@ -1017,6 +1017,281 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
#include "ceval_macros.h"
/* Helper functions to keep the size of the largest uops down */
PyObject *
_Py_VectorCall_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args,
_PyStackRef kwnames)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
int positional_args = total_args;
if (kwnames_o != NULL) {
positional_args -= (int)PyTuple_GET_SIZE(kwnames_o);
}
res = PyObject_Vectorcall(
callable_o, args_o,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames_o);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
PyStackRef_XCLOSE(kwnames);
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}
PyObject *
_Py_BuiltinCallFast_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
res = _PyCFunctionFast_CAST(cfunc)(
PyCFunction_GET_SELF(callable_o),
args_o,
total_args
);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}
PyObject *
_Py_BuiltinCallFastWithKeywords_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyCFunctionFastWithKeywords cfunc =
_PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o));
res = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}
PyObject *
_PyCallMethodDescriptorFast_StackRefSteal(
_PyStackRef callable,
PyMethodDef *meth,
PyObject *self,
_PyStackRef *arguments,
int total_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth);
assert(self == PyStackRef_AsPyObjectBorrow(arguments[0]));
PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth);
res = cfunc(self, (args_o + 1), total_args - 1);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}
PyObject *
_PyCallMethodDescriptorFastWithKeywords_StackRefSteal(
_PyStackRef callable,
PyMethodDef *meth,
PyObject *self,
_PyStackRef *arguments,
int total_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth);
assert(self == PyStackRef_AsPyObjectBorrow(arguments[0]));
PyCFunctionFastWithKeywords cfunc =
_PyCFunctionFastWithKeywords_CAST(meth->ml_meth);
res = cfunc(self, (args_o + 1), total_args-1, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}
PyObject *
_Py_CallBuiltinClass_StackRefSteal(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(callable);
res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}
PyObject *
_Py_BuildString_StackRefSteal(
_PyStackRef *arguments,
int total_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
res = _PyUnicode_JoinArray(&_Py_STR(empty), args_o, total_args);
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
return res;
}
PyObject *
_Py_BuildMap_StackRefSteal(
_PyStackRef *arguments,
int half_args)
{
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, half_args*2, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
}
res = _PyDict_FromItems(
args_o, 2,
args_o+1, 2,
half_args
);
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = half_args*2-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
return res;
}
#ifdef Py_DEBUG
void
_Py_assert_within_stack_bounds(
_PyInterpreterFrame *frame, _PyStackRef *stack_pointer,
const char *filename, int lineno
) {
if (frame->owner == FRAME_OWNED_BY_INTERPRETER) {
return;
}
int level = (int)(stack_pointer - _PyFrame_Stackbase(frame));
if (level < 0) {
printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno);
fflush(stdout);
abort();
}
int size = _PyFrame_GetCode(frame)->co_stacksize;
if (level > size) {
printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno);
fflush(stdout);
abort();
}
}
#endif
int _Py_CheckRecursiveCallPy(
PyThreadState *tstate)
{
@ -1078,11 +1353,12 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject
if (result == NULL) {
return NULL;
}
result++;
}
else {
result = scratch;
}
result++;
result[0] = NULL; /* Keep GCC happy */
for (int i = 0; i < nargs; i++) {
result[i] = PyStackRef_AsPyObjectBorrow(input[i]);
}
@ -1097,6 +1373,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch)
}
}
#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif
#if _Py_TIER2
// 0 for success, -1 for error.
static int

2
Python/ceval_macros.h
View file

@ -458,7 +458,7 @@ do { \
#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \
/* +1 because vectorcall might use -1 to write self */ \
PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \
PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1);
PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp);
#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \
/* +1 because we +1 previously */ \

1094
Python/executor_cases.c.h generated
View file

File diff suppressed because it is too large Load diff

1070
Python/generated_cases.c.h generated
View file

File diff suppressed because it is too large Load diff

4
Python/jit.c
View file

@ -60,6 +60,10 @@ jit_error(const char *message)
static unsigned char *
jit_alloc(size_t size)
{
if (size > PY_MAX_JIT_CODE_SIZE) {
jit_error("code too big; refactor bytecodes.c to keep uop size down, or reduce maximum trace length.");
return NULL;
}
assert(size);
assert(size % get_page_size() == 0);
#ifdef MS_WINDOWS

30
Python/optimizer_analysis.c
View file

@ -278,6 +278,36 @@ get_co_name(JitOptContext *ctx, int index)
return PyTuple_GET_ITEM(get_current_code_object(ctx)->co_names, index);
}
#ifdef Py_DEBUG
void
_Py_opt_assert_within_stack_bounds(
_Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer,
const char *filename, int lineno
) {
if (frame->code == ((PyCodeObject *)&_Py_InitCleanup)) {
return;
}
int level = (int)(stack_pointer - frame->stack);
if (level < 0) {
printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno);
fflush(stdout);
abort();
}
int size = (int)(frame->stack_len);
if (level > size) {
printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno);
fflush(stdout);
abort();
}
}
#endif
#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_opt_assert_within_stack_bounds(ctx->frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif
// TODO (gh-134584) generate most of this table automatically
const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = {
[_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS,

346
Python/optimizer_cases.c.h generated
View file

File diff suppressed because it is too large Load diff

2
Tools/cases_generator/stack.py
View file

@ -296,7 +296,7 @@ def _save_physical_sp(self, out: CWriter) -> None:
diff = self.logical_sp - self.physical_sp
out.start_line()
out.emit(f"stack_pointer += {diff.to_c()};\n")
out.emit(f"assert(WITHIN_STACK_BOUNDS());\n")
out.emit(f"ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);\n")
self.physical_sp = self.logical_sp
self._print(out)

6
Tools/jit/template.c
View file

@ -86,6 +86,12 @@ do { \
#define TIER_TWO 2
#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif
__attribute__((preserve_none)) _Py_CODEUNIT *
_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate)
{