gh-148171: Convert CALL_BUILTIN_CLASS to leave arguments on the stack (gh-148381)

This commit is contained in:
Donghee Na 2026-04-11 23:01:25 +09:00 committed by GitHub
parent 9831dea3bf
commit a71b043356
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 87 additions and 60 deletions

View file

@ -461,7 +461,7 @@ _PyCallMethodDescriptorFastWithKeywords_StackRef(
int total_args);
PyAPI_FUNC(PyObject *)
_Py_CallBuiltinClass_StackRefSteal(
_Py_CallBuiltinClass_StackRef(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args);

View file

@ -1123,7 +1123,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
[CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG },
[CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG },
@ -1374,7 +1374,7 @@ _PyOpcode_macro_expansion[256] = {
[CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 5, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } },
[CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } },
[CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } },
[CALL_BUILTIN_CLASS] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },

View file

@ -499,7 +499,7 @@ extern "C" {
#define _BUILD_STRING_r01 709
#define _BUILD_TEMPLATE_r21 710
#define _BUILD_TUPLE_r01 711
#define _CALL_BUILTIN_CLASS_r01 712
#define _CALL_BUILTIN_CLASS_r00 712
#define _CALL_BUILTIN_FAST_r00 713
#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 714
#define _CALL_BUILTIN_O_r03 715

View file

@ -307,7 +307,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG,
[_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_CALLABLE_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_CALLABLE_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_CALLABLE_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
@ -2877,7 +2877,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = {
[_CALL_BUILTIN_CLASS] = {
.best = { 0, 0, 0, 0 },
.entries = {
{ 1, 0, _CALL_BUILTIN_CLASS_r01 },
{ 0, 0, _CALL_BUILTIN_CLASS_r00 },
{ -1, -1, -1 },
{ -1, -1, -1 },
{ -1, -1, -1 },
@ -4348,7 +4348,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = {
[_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME,
[_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK,
[_GUARD_CALLABLE_BUILTIN_CLASS_r00] = _GUARD_CALLABLE_BUILTIN_CLASS,
[_CALL_BUILTIN_CLASS_r01] = _CALL_BUILTIN_CLASS,
[_CALL_BUILTIN_CLASS_r00] = _CALL_BUILTIN_CLASS,
[_GUARD_CALLABLE_BUILTIN_O_r00] = _GUARD_CALLABLE_BUILTIN_O,
[_CALL_BUILTIN_O_r03] = _CALL_BUILTIN_O,
[_GUARD_CALLABLE_BUILTIN_FAST_r00] = _GUARD_CALLABLE_BUILTIN_FAST,
@ -4736,7 +4736,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {
[_BUILD_TUPLE] = "_BUILD_TUPLE",
[_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01",
[_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS",
[_CALL_BUILTIN_CLASS_r01] = "_CALL_BUILTIN_CLASS_r01",
[_CALL_BUILTIN_CLASS_r00] = "_CALL_BUILTIN_CLASS_r00",
[_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST",
[_CALL_BUILTIN_FAST_r00] = "_CALL_BUILTIN_FAST_r00",
[_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS",
@ -6408,7 +6408,7 @@ int _PyUop_num_popped(int opcode, int oparg)
case _GUARD_CALLABLE_BUILTIN_CLASS:
return 0;
case _CALL_BUILTIN_CLASS:
return 2 + oparg;
return 0;
case _GUARD_CALLABLE_BUILTIN_O:
return 0;
case _CALL_BUILTIN_O:

View file

@ -2276,7 +2276,7 @@
_PyStackRef callable;
_PyStackRef self_or_null;
_PyStackRef *args;
_PyStackRef res;
_PyStackRef value;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _GUARD_CALLABLE_BUILTIN_CLASS
@ -2307,23 +2307,39 @@
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal(
PyObject *res_o = _Py_CallBuiltinClass_StackRef(
callable,
arguments,
total_args);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = callable;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _POP_TOP_OPARG
{
args = &stack_pointer[-oparg];
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef_CloseStack(args, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _POP_TOP
{
value = self_or_null;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _CHECK_PERIODIC_AT_END
{
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);

View file

@ -4572,7 +4572,7 @@ dummy_func(
EXIT_IF(tp->tp_vectorcall == NULL);
}
op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- res)) {
op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
int total_args = oparg;
_PyStackRef *arguments = args;
if (!PyStackRef_IsNull(self_or_null)) {
@ -4580,15 +4580,16 @@ dummy_func(
total_args++;
}
STAT_INC(CALL, hit);
PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal(
PyObject *res_o = _Py_CallBuiltinClass_StackRef(
callable,
arguments,
total_args);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
if (res_o == NULL) {
ERROR_NO_POP();
}
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
PyStackRef_CLOSE(temp);
}
macro(CALL_BUILTIN_CLASS) =
@ -4597,6 +4598,8 @@ dummy_func(
unused/2 +
_GUARD_CALLABLE_BUILTIN_CLASS +
_CALL_BUILTIN_CLASS +
_POP_TOP_OPARG +
POP_TOP +
_CHECK_PERIODIC_AT_END;
op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {

View file

@ -894,7 +894,7 @@ _PyCallMethodDescriptorFastWithKeywords_StackRef(
}
PyObject *
_Py_CallBuiltinClass_StackRefSteal(
_Py_CallBuiltinClass_StackRef(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args)
@ -902,22 +902,12 @@ _Py_CallBuiltinClass_StackRefSteal(
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
return NULL;
}
PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(callable);
res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}

View file

@ -16450,13 +16450,12 @@
break;
}
case _CALL_BUILTIN_CLASS_r01: {
case _CALL_BUILTIN_CLASS_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
@ -16469,24 +16468,25 @@
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal(
PyObject *res_o = _Py_CallBuiltinClass_StackRef(
callable,
arguments,
total_args);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
res = PyStackRef_FromPyObjectSteal(res_o);
_tos_cache0 = res;
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = callable;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
_tos_cache0 = PyStackRef_ZERO_BITS;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}

View file

@ -2276,7 +2276,7 @@
_PyStackRef callable;
_PyStackRef self_or_null;
_PyStackRef *args;
_PyStackRef res;
_PyStackRef value;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _GUARD_CALLABLE_BUILTIN_CLASS
@ -2307,23 +2307,39 @@
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal(
PyObject *res_o = _Py_CallBuiltinClass_StackRef(
callable,
arguments,
total_args);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = callable;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _POP_TOP_OPARG
{
args = &stack_pointer[-oparg];
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef_CloseStack(args, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _POP_TOP
{
value = self_or_null;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _CHECK_PERIODIC_AT_END
{
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);

View file

@ -1364,6 +1364,10 @@ dummy_func(void) {
callable = sym_new_not_null(ctx);
}
op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
callable = sym_new_not_null(ctx);
}
op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
PyObject *callable_o = sym_get_const(ctx, callable);
if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) &&

View file

@ -3950,12 +3950,10 @@
}
case _CALL_BUILTIN_CLASS: {
JitOptRef res;
res = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(-1 - oparg);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JitOptRef callable;
callable = stack_pointer[-2 - oparg];
callable = sym_new_not_null(ctx);
stack_pointer[-2 - oparg] = callable;
break;
}