Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-21 03:10:52 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-148171: Convert CALL_BUILTIN_FAST to leave inputs on the stack for refcount elimination in JIT (GH-148172)

Browse source
This commit is contained in:
Ken Jin 2026-04-10 23:11:18 +08:00 • committed by GitHub
parent aea0b91d65
commit 266247c9a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 1199 additions and 1103 deletions
Download patch file Download diff file Expand all files Collapse all files

22
Python/bytecodes.c
View file

@ -386,6 +386,11 @@ dummy_func(
PyStackRef_CLOSE(nos);
}
op(_POP_TOP_OPARG, (args[oparg] -- )) {
_PyStackRef_CloseStack(args, oparg);
DEAD(args);
}
pure inst(PUSH_NULL, (-- res)) {
res = PyStackRef_NULL;
}
@ -4646,7 +4651,7 @@ dummy_func(
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL);
}
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) {
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
/* Builtin METH_FASTCALL functions, without keywords */
int total_args = oparg;
_PyStackRef *arguments = args;
@ -4655,16 +4660,17 @@ dummy_func(
total_args++;
}
STAT_INC(CALL, hit);
PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal(
PyObject *res_o = _Py_BuiltinCallFast_StackRef(
callable,
arguments,
total_args
);
DEAD(args);
DEAD(self_or_null);
DEAD(callable);
ERROR_IF(res_o == NULL);
res = PyStackRef_FromPyObjectSteal(res_o);
if (res_o == NULL) {
ERROR_NO_POP();
}
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
PyStackRef_CLOSE(temp);
}
macro(CALL_BUILTIN_FAST) =
@ -4673,6 +4679,8 @@ dummy_func(
unused/2 +
_GUARD_CALLABLE_BUILTIN_FAST +
_CALL_BUILTIN_FAST +
_POP_TOP_OPARG +
POP_TOP +
_CHECK_PERIODIC_AT_END;
op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {

14
Python/ceval.c
View file

@ -809,7 +809,7 @@ _Py_VectorCallInstrumentation_StackRefSteal(
}
PyObject *
_Py_BuiltinCallFast_StackRefSteal(
_Py_BuiltinCallFast_StackRef(
_PyStackRef callable,
_PyStackRef *arguments,
int total_args)
@ -817,8 +817,7 @@ _Py_BuiltinCallFast_StackRefSteal(
PyObject *res;
STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o);
if (CONVERSION_FAILED(args_o)) {
res = NULL;
goto cleanup;
return NULL;
}
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o);
@ -829,15 +828,6 @@ _Py_BuiltinCallFast_StackRefSteal(
);
STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
assert((res != NULL) ^ (PyErr_Occurred() != NULL));
cleanup:
// arguments is a pointer into the GC visible stack,
// so we must NULL out values as we clear them.
for (int i = total_args-1; i >= 0; i--) {
_PyStackRef tmp = arguments[i];
arguments[i] = PyStackRef_NULL;
PyStackRef_CLOSE(tmp);
}
PyStackRef_CLOSE(callable);
return res;
}

39
Python/executor_cases.c.h generated
View file

@ -2525,6 +2525,25 @@
break;
}
case _POP_TOP_OPARG_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef *args;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef_CloseStack(args, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
_tos_cache0 = PyStackRef_ZERO_BITS;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(0);
stack_pointer += -oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _PUSH_NULL_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
@ -16591,13 +16610,12 @@
break;
}
case _CALL_BUILTIN_FAST_r01: {
case _CALL_BUILTIN_FAST_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef *args;
_PyStackRef self_or_null;
_PyStackRef callable;
_PyStackRef res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
@ -16610,25 +16628,26 @@
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal(
PyObject *res_o = _Py_BuiltinCallFast_StackRef(
callable,
arguments,
total_args
);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
res = PyStackRef_FromPyObjectSteal(res_o);
_tos_cache0 = res;
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = callable;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
_tos_cache0 = PyStackRef_ZERO_BITS;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}

32
Python/generated_cases.c.h generated
View file

@ -2345,7 +2345,7 @@
_PyStackRef callable;
_PyStackRef self_or_null;
_PyStackRef *args;
_PyStackRef res;
_PyStackRef value;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _GUARD_CALLABLE_BUILTIN_FAST
@ -2375,24 +2375,40 @@
}
STAT_INC(CALL, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal(
PyObject *res_o = _Py_BuiltinCallFast_StackRef(
callable,
arguments,
total_args
);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
stack_pointer += -2 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
_PyStackRef temp = callable;
callable = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2 - oparg] = callable;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _POP_TOP_OPARG
{
args = &stack_pointer[-oparg];
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef_CloseStack(args, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _POP_TOP
{
value = self_or_null;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(value);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _CHECK_PERIODIC_AT_END
{
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);

23
Python/optimizer_analysis.c
View file

@ -405,6 +405,29 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter *dependencies, _PyUOpInstruction
return sym_new_not_null(ctx);
}
static void
optimize_pop_top(JitOptContext *ctx, _PyUOpInstruction *this_instr, JitOptRef value)
{
PyTypeObject *typ = sym_get_type(value);
if (PyJitRef_IsBorrowed(value) ||
sym_is_immortal(PyJitRef_Unwrap(value)) ||
sym_is_null(value)) {
ADD_OP(_POP_TOP_NOP, 0, 0);
}
else if (typ == &PyLong_Type) {
ADD_OP(_POP_TOP_INT, 0, 0);
}
else if (typ == &PyFloat_Type) {
ADD_OP(_POP_TOP_FLOAT, 0, 0);
}
else if (typ == &PyUnicode_Type) {
ADD_OP(_POP_TOP_UNICODE, 0, 0);
}
else {
ADD_OP(_POP_TOP, 0, 0);
}
}
/* Look up name via super (normal case from supercheck where
su_obj_type = Py_TYPE(obj)). */
static JitOptRef

26
Python/optimizer_bytecodes.c
View file

@ -763,22 +763,14 @@ dummy_func(void) {
value = PyJitRef_Borrow(sym_new_const(ctx, ptr));
}
op(_POP_TOP_OPARG, (args[oparg] --)) {
for (int i = oparg-1; i >= 0; i--) {
optimize_pop_top(ctx, this_instr, args[i]);
}
}
op(_POP_TOP, (value -- )) {
PyTypeObject *typ = sym_get_type(value);
if (PyJitRef_IsBorrowed(value) ||
sym_is_immortal(PyJitRef_Unwrap(value)) ||
sym_is_null(value)) {
ADD_OP(_POP_TOP_NOP, 0, 0);
}
else if (typ == &PyLong_Type) {
ADD_OP(_POP_TOP_INT, 0, 0);
}
else if (typ == &PyFloat_Type) {
ADD_OP(_POP_TOP_FLOAT, 0, 0);
}
else if (typ == &PyUnicode_Type) {
ADD_OP(_POP_TOP_UNICODE, 0, 0);
}
optimize_pop_top(ctx, this_instr, value);
}
op(_POP_TOP_INT, (value --)) {
@ -1348,6 +1340,10 @@ dummy_func(void) {
}
}
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
callable = sym_new_not_null(ctx);
}
op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
PyObject *callable_o = sym_get_const(ctx, callable);
if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) &&

38
Python/optimizer_cases.c.h generated
View file

@ -115,21 +115,7 @@
case _POP_TOP: {
JitOptRef value;
value = stack_pointer[-1];
PyTypeObject *typ = sym_get_type(value);
if (PyJitRef_IsBorrowed(value) ||
sym_is_immortal(PyJitRef_Unwrap(value)) ||
sym_is_null(value)) {
ADD_OP(_POP_TOP_NOP, 0, 0);
}
else if (typ == &PyLong_Type) {
ADD_OP(_POP_TOP_INT, 0, 0);
}
else if (typ == &PyFloat_Type) {
ADD_OP(_POP_TOP_FLOAT, 0, 0);
}
else if (typ == &PyUnicode_Type) {
ADD_OP(_POP_TOP_UNICODE, 0, 0);
}
optimize_pop_top(ctx, this_instr, value);
CHECK_STACK_BOUNDS(-1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@ -186,6 +172,18 @@
break;
}
case _POP_TOP_OPARG: {
JitOptRef *args;
args = &stack_pointer[-oparg];
for (int i = oparg-1; i >= 0; i--) {
optimize_pop_top(ctx, this_instr, args[i]);
}
CHECK_STACK_BOUNDS(-oparg);
stack_pointer += -oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _PUSH_NULL: {
JitOptRef res;
res = PyJitRef_Borrow(sym_new_null(ctx));
@ -4001,12 +3999,10 @@
}
case _CALL_BUILTIN_FAST: {
JitOptRef res;
res = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(-1 - oparg);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JitOptRef callable;
callable = stack_pointer[-2 - oparg];
callable = sym_new_not_null(ctx);
stack_pointer[-2 - oparg] = callable;
break;
}