Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-25 21:30:45 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-142982: Specialize CALL_FUNCTION_EX (GH-143391)

Browse source
This commit is contained in:
Ken Jin 2026-01-07 04:34:08 +08:00 • committed by GitHub
parent ff7d1cec41
commit df355348f0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 2074 additions and 1188 deletions
Download patch file Download diff file Expand all files Collapse all files

89
Python/bytecodes.c
View file

@ -4794,6 +4794,11 @@ dummy_func(
_CALL_KW_NON_PY +
_CHECK_PERIODIC_AT_END;
family(CALL_FUNCTION_EX, INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX) = {
CALL_EX_PY,
CALL_EX_NON_PY_GENERAL,
};
op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs -- func, unused, callargs, kwargs)) {
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
@ -4872,8 +4877,8 @@ dummy_func(
if (new_frame == NULL) {
ERROR_NO_POP();
}
assert(INSTRUCTION_SIZE == 1);
frame->return_offset = 1;
assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
frame->return_offset = INSTRUCTION_SIZE;
DISPATCH_INLINED(new_frame);
}
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
@ -4890,12 +4895,92 @@ dummy_func(
result = PyStackRef_FromPyObjectSteal(result_o);
}
specializing op(_SPECIALIZE_CALL_FUNCTION_EX, (counter/1, func, unused, unused, unused -- func, unused, unused, unused)) {
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_CallFunctionEx(func, next_instr);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CALL_FUNCTION_EX);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
macro(CALL_FUNCTION_EX) =
_SPECIALIZE_CALL_FUNCTION_EX +
_MAKE_CALLARGS_A_TUPLE +
_DO_CALL_FUNCTION_EX +
_CHECK_PERIODIC_AT_END;
op(_CHECK_IS_PY_CALLABLE_EX, (func_st, unused, unused, unused -- func_st, unused, unused, unused)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
EXIT_IF(Py_TYPE(func) != &PyFunction_Type);
EXIT_IF(((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall);
}
op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st);
assert(PyTuple_CheckExact(callargs));
assert(Py_TYPE(func) == &PyFunction_Type);
assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
tstate, func_st, locals,
nargs, callargs, kwargs, frame);
INPUTS_DEAD();
SYNC_SP();
if (new_frame == NULL) {
ERROR_NO_POP();
}
ex_frame = PyStackRef_Wrap(new_frame);
}
macro(CALL_EX_PY) =
unused/1 +
_CHECK_PEP_523 +
_MAKE_CALLARGS_A_TUPLE +
_CHECK_IS_PY_CALLABLE_EX +
_PY_FRAME_EX +
_SAVE_RETURN_OFFSET +
_PUSH_FRAME;
op(_CHECK_IS_NOT_PY_CALLABLE_EX, (func_st, unused, unused, unused -- func_st, unused, unused, unused)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
EXIT_IF(Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
}
op(_CALL_FUNCTION_EX_NON_PY_GENERAL, (func_st, null, callargs_st, kwargs_st -- result)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
(void)null;
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
PyObject *result_o = PyObject_Call(func, callargs, kwargs);
PyStackRef_XCLOSE(kwargs_st);
PyStackRef_CLOSE(callargs_st);
DEAD(null);
PyStackRef_CLOSE(func_st);
ERROR_IF(result_o == NULL);
result = PyStackRef_FromPyObjectSteal(result_o);
}
macro(CALL_EX_NON_PY_GENERAL) =
unused/1 +
_CHECK_IS_NOT_PY_CALLABLE_EX +
_MAKE_CALLARGS_A_TUPLE +
_CALL_FUNCTION_EX_NON_PY_GENERAL +
_CHECK_PERIODIC_AT_END;
macro(INSTRUMENTED_CALL_FUNCTION_EX) =
unused/1 +
_MAKE_CALLARGS_A_TUPLE +
_DO_CALL_FUNCTION_EX +
_CHECK_PERIODIC_AT_END;

5
Python/ceval.c
View file

@ -329,9 +329,6 @@ static void monitor_throw(PyThreadState *tstate,
_Py_CODEUNIT *instr);
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);
#ifdef HAVE_ERRNO_H
#include <errno.h>
@ -2435,7 +2432,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func,
/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict.
Steals references to func, callargs and kwargs.
*/
static _PyInterpreterFrame *
_PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous)
{

317
Python/executor_cases.c.h generated
View file

@ -15694,6 +15694,323 @@
/* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _CHECK_IS_PY_CALLABLE_EX_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
func_st = stack_pointer[-4];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = stack_pointer[-1];
_tos_cache1 = stack_pointer[-2];
_tos_cache0 = stack_pointer[-3];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_PY_CALLABLE_EX_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
func_st = stack_pointer[-3];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_0;
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = stack_pointer[-2];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_PY_CALLABLE_EX_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
func_st = stack_pointer[-2];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_1;
_tos_cache1 = _stack_item_0;
_tos_cache0 = stack_pointer[-1];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_PY_CALLABLE_EX_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _PY_FRAME_EX_r31: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef kwargs_st;
_PyStackRef callargs_st;
_PyStackRef func_st;
_PyStackRef ex_frame;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
kwargs_st = _stack_item_2;
callargs_st = _stack_item_1;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st);
assert(PyTuple_CheckExact(callargs));
assert(Py_TYPE(func) == &PyFunction_Type);
assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
stack_pointer[0] = _stack_item_0;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
tstate, func_st, locals,
nargs, callargs, kwargs, frame);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
if (new_frame == NULL) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
ex_frame = PyStackRef_Wrap(new_frame);
_tos_cache0 = ex_frame;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
func_st = stack_pointer[-4];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = stack_pointer[-1];
_tos_cache1 = stack_pointer[-2];
_tos_cache0 = stack_pointer[-3];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
func_st = stack_pointer[-3];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_0;
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = stack_pointer[-2];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
func_st = stack_pointer[-2];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_1;
_tos_cache1 = _stack_item_0;
_tos_cache0 = stack_pointer[-1];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CALL_FUNCTION_EX_NON_PY_GENERAL_r31: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef kwargs_st;
_PyStackRef callargs_st;
_PyStackRef null;
_PyStackRef func_st;
_PyStackRef result;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
kwargs_st = _stack_item_2;
callargs_st = _stack_item_1;
null = _stack_item_0;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
(void)null;
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
stack_pointer[0] = null;
stack_pointer[1] = callargs_st;
stack_pointer[2] = kwargs_st;
stack_pointer += 3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(kwargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(callargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
result = PyStackRef_FromPyObjectSteal(result_o);
_tos_cache0 = result;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _MAKE_FUNCTION_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());

264
Python/generated_cases.c.h generated
View file

@ -2499,16 +2499,245 @@
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
TARGET(CALL_EX_NON_PY_GENERAL) {
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_FUNCTION_EX;
int opcode = CALL_EX_NON_PY_GENERAL;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(CALL_EX_NON_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size");
_PyStackRef func_st;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef null;
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef result;
/* Skip 1 cache entry */
// _CHECK_IS_NOT_PY_CALLABLE_EX
{
func_st = stack_pointer[-4];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
}
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
func = func_st;
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
JUMP_TO_LABEL(error);
}
_PyStackRef temp = callargs;
callargs = PyStackRef_FromPyObjectSteal(tuple_o);
stack_pointer[-2] = callargs;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
}
// _CALL_FUNCTION_EX_NON_PY_GENERAL
{
kwargs_st = stack_pointer[-1];
callargs_st = callargs;
null = stack_pointer[-3];
func_st = func;
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
(void)null;
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
stack_pointer[-2] = callargs_st;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(kwargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(callargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
// _CHECK_PERIODIC_AT_END
{
stack_pointer[0] = result;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
DISPATCH();
}
TARGET(CALL_EX_PY) {
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_EX_PY;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CALL_EX_PY);
static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size");
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef func_st;
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef ex_frame;
_PyStackRef new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
}
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
func = stack_pointer[-4];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
JUMP_TO_LABEL(error);
}
_PyStackRef temp = callargs;
callargs = PyStackRef_FromPyObjectSteal(tuple_o);
stack_pointer[-2] = callargs;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
}
// _CHECK_IS_PY_CALLABLE_EX
{
func_st = func;
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
}
// _PY_FRAME_EX
{
kwargs_st = stack_pointer[-1];
callargs_st = callargs;
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st);
assert(PyTuple_CheckExact(callargs));
assert(Py_TYPE(func) == &PyFunction_Type);
assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
tstate, func_st, locals,
nargs, callargs, kwargs, frame);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
if (new_frame == NULL) {
JUMP_TO_LABEL(error);
}
ex_frame = PyStackRef_Wrap(new_frame);
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
new_frame = ex_frame;
assert(tstate->interp->eval_frame == NULL);
_PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(temp->previous == frame || temp->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = temp;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_FUNCTION_EX;
(void)(opcode);
#endif
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CALL_FUNCTION_EX);
PREDICTED_CALL_FUNCTION_EX:;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
opcode = CALL_FUNCTION_EX;
_PyStackRef func;
_PyStackRef callargs;
@ -2517,10 +2746,26 @@
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef result;
// _SPECIALIZE_CALL_FUNCTION_EX
{
func = stack_pointer[-4];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_CallFunctionEx(func, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CALL_FUNCTION_EX);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
func = stack_pointer[-4];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -2618,8 +2863,8 @@
if (new_frame == NULL) {
JUMP_TO_LABEL(error);
}
assert( 1u == 1);
frame->return_offset = 1;
assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
frame->return_offset = 2u ;
DISPATCH_INLINED(new_frame);
}
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
@ -6247,7 +6492,7 @@
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
opcode = INSTRUMENTED_CALL_FUNCTION_EX;
_PyStackRef func;
@ -6257,6 +6502,7 @@
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef result;
/* Skip 1 cache entry */
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
@ -6358,8 +6604,8 @@
if (new_frame == NULL) {
JUMP_TO_LABEL(error);
}
assert( 1u == 1);
frame->return_offset = 1;
assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
frame->return_offset = 2u ;
DISPATCH_INLINED(new_frame);
}
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);

18
Python/opcode_targets.h generated
View file

@ -151,6 +151,8 @@ static void *opcode_targets_table[256] = {
&&TARGET_CALL_BUILTIN_FAST,
&&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
&&TARGET_CALL_BUILTIN_O,
&&TARGET_CALL_EX_NON_PY_GENERAL,
&&TARGET_CALL_EX_PY,
&&TARGET_CALL_ISINSTANCE,
&&TARGET_CALL_KW_BOUND_METHOD,
&&TARGET_CALL_KW_NON_PY,
@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_POP_ITER,
&&TARGET_INSTRUMENTED_END_SEND,
@ -470,8 +470,8 @@ static void *opcode_tracing_targets_table[256] = {
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
@ -565,6 +565,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS);
@ -804,6 +806,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST,
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS,
[CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O,
[CALL_EX_NON_PY_GENERAL] = _TAIL_CALL_CALL_EX_NON_PY_GENERAL,
[CALL_EX_PY] = _TAIL_CALL_CALL_EX_PY,
[CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX,
[CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1,
[CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2,
@ -1003,8 +1007,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[211] = _TAIL_CALL_UNKNOWN_OPCODE,
[212] = _TAIL_CALL_UNKNOWN_OPCODE,
[213] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,
@ -1062,6 +1064,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD,
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD,
[CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD,
[CALL_EX_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD,
[CALL_EX_PY] = _TAIL_CALL_TRACE_RECORD,
[CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD,
[CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD,
[CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD,
@ -1261,8 +1265,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[211] = _TAIL_CALL_UNKNOWN_OPCODE,
[212] = _TAIL_CALL_UNKNOWN_OPCODE,
[213] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,

14
Python/optimizer_bytecodes.c
View file

@ -837,6 +837,17 @@ dummy_func(void) {
new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0));
}
op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) {
assert((this_instr + 2)->opcode == _PUSH_FRAME);
PyCodeObject *co = get_code_with_logging((this_instr + 2));
if (co == NULL) {
ctx->done = true;
break;
}
ex_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0));
}
op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
(void)type_version;
(void)args;
@ -1000,8 +1011,7 @@ dummy_func(void) {
ctx->frame->func = func;
}
// Fixed calls don't need IP guards.
if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET ||
(this_instr-1)->opcode == _CREATE_INIT_FRAME) {
if ((this_instr-1)->opcode == _CREATE_INIT_FRAME) {
assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME);
REPLACE_OP(this_instr+1, _NOP, 0, 0);
}

37
Python/optimizer_cases.c.h generated
View file

@ -2640,8 +2640,7 @@
PyFunctionObject *func = (PyFunctionObject *)operand;
ctx->frame->func = func;
}
if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET ||
(this_instr-1)->opcode == _CREATE_INIT_FRAME) {
if ((this_instr-1)->opcode == _CREATE_INIT_FRAME) {
assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME);
REPLACE_OP(this_instr+1, _NOP, 0, 0);
}
@ -3112,6 +3111,40 @@
/* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
case _CHECK_IS_PY_CALLABLE_EX: {
break;
}
case _PY_FRAME_EX: {
JitOptRef ex_frame;
assert((this_instr + 2)->opcode == _PUSH_FRAME);
PyCodeObject *co = get_code_with_logging((this_instr + 2));
if (co == NULL) {
ctx->done = true;
break;
}
ex_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0));
CHECK_STACK_BOUNDS(-3);
stack_pointer[-4] = ex_frame;
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX: {
break;
}
case _CALL_FUNCTION_EX_NON_PY_GENERAL: {
JitOptRef result;
result = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(-3);
stack_pointer[-4] = result;
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _MAKE_FUNCTION: {
JitOptRef func;
func = sym_new_not_null(ctx);

22
Python/specialize.c
View file

@ -2587,6 +2587,28 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr)
unspecialize(instr);
}
Py_NO_INLINE void
_Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr)
{
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
if (Py_TYPE(func) == &PyFunction_Type &&
((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
if (_PyInterpreterState_GET()->eval_frame) {
goto failure;
}
specialize(instr, CALL_EX_PY);
return;
}
specialize(instr, CALL_EX_NON_PY_GENERAL);
return;
failure:
unspecialize(instr);
}
#ifdef Py_STATS
static int
to_bool_fail_kind(PyObject *value)