mirror of
https://github.com/python/cpython.git
synced 2026-04-15 00:00:57 +00:00
gh-131798: JIT: split call method and call builtin opcodes into smaller uops (#146463)
This commit is contained in:
Kumar Aditya
GitHub
parent
0e543055b0
commit
bfdaa3c565
10 changed files with 2324 additions and 1540 deletions
|
|
@ -4479,18 +4479,24 @@ dummy_func(
|
|||
_CALL_BUILTIN_CLASS +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
EXIT_IF(!PyCFunction_CheckExact(callable_o));
|
||||
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O);
|
||||
int total_args = oparg;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args != 1);
|
||||
}
|
||||
|
||||
op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) {
|
||||
/* Builtin METH_O functions */
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
|
||||
int total_args = oparg;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
args--;
|
||||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args != 1);
|
||||
EXIT_IF(!PyCFunction_CheckExact(callable_o));
|
||||
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_O);
|
||||
// CPython promises to check all non-vectorcall function calls.
|
||||
EXIT_IF(_Py_ReachedRecursionLimit(tstate));
|
||||
STAT_INC(CALL, hit);
|
||||
|
|
@ -4512,11 +4518,18 @@ dummy_func(
|
|||
_RECORD_CALLABLE +
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_BUILTIN_O +
|
||||
_CALL_BUILTIN_O +
|
||||
POP_TOP +
|
||||
POP_TOP +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
EXIT_IF(!PyCFunction_CheckExact(callable_o));
|
||||
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL);
|
||||
}
|
||||
|
||||
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) {
|
||||
/* Builtin METH_FASTCALL functions, without keywords */
|
||||
int total_args = oparg;
|
||||
|
|
@ -4525,9 +4538,6 @@ dummy_func(
|
|||
arguments--;
|
||||
total_args++;
|
||||
}
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
EXIT_IF(!PyCFunction_CheckExact(callable_o));
|
||||
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL);
|
||||
STAT_INC(CALL, hit);
|
||||
PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal(
|
||||
callable,
|
||||
|
|
@ -4545,9 +4555,16 @@ dummy_func(
|
|||
_RECORD_CALLABLE +
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_BUILTIN_FAST +
|
||||
_CALL_BUILTIN_FAST +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
EXIT_IF(!PyCFunction_CheckExact(callable_o));
|
||||
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS));
|
||||
}
|
||||
|
||||
op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
|
||||
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
|
||||
int total_args = oparg;
|
||||
|
|
@ -4556,9 +4573,6 @@ dummy_func(
|
|||
arguments--;
|
||||
total_args++;
|
||||
}
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
EXIT_IF(!PyCFunction_CheckExact(callable_o));
|
||||
EXIT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS));
|
||||
STAT_INC(CALL, hit);
|
||||
PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args);
|
||||
DEAD(args);
|
||||
|
|
@ -4572,6 +4586,7 @@ dummy_func(
|
|||
_RECORD_CALLABLE +
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS +
|
||||
_CALL_BUILTIN_FAST_WITH_KEYWORDS +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
|
|
@ -4674,29 +4689,35 @@ dummy_func(
|
|||
none = PyStackRef_None;
|
||||
}
|
||||
|
||||
op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
EXIT_IF(method->d_method->ml_flags != METH_O);
|
||||
int total_args = oparg;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args != 2);
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(
|
||||
PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null);
|
||||
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
|
||||
}
|
||||
|
||||
op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
|
||||
int total_args = oparg;
|
||||
_PyStackRef *arguments = args;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
arguments--;
|
||||
total_args++;
|
||||
}
|
||||
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
EXIT_IF(total_args != 2);
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
PyMethodDef *meth = method->d_method;
|
||||
EXIT_IF(meth->ml_flags != METH_O);
|
||||
// CPython promises to check all non-vectorcall function calls.
|
||||
EXIT_IF(_Py_ReachedRecursionLimit(tstate));
|
||||
_PyStackRef arg_stackref = arguments[1];
|
||||
_PyStackRef self_stackref = arguments[0];
|
||||
EXIT_IF(!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref),
|
||||
method->d_common.d_type));
|
||||
STAT_INC(CALL, hit);
|
||||
PyCFunction cfunc = meth->ml_meth;
|
||||
PyCFunction cfunc = method->d_method->ml_meth;
|
||||
PyObject *res_o = _PyCFunction_TrampolineCall(cfunc,
|
||||
PyStackRef_AsPyObjectBorrow(self_stackref),
|
||||
PyStackRef_AsPyObjectBorrow(arg_stackref));
|
||||
|
|
@ -4716,15 +4737,18 @@ dummy_func(
|
|||
_RECORD_CALLABLE +
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_METHOD_DESCRIPTOR_O +
|
||||
_CALL_METHOD_DESCRIPTOR_O +
|
||||
POP_TOP +
|
||||
POP_TOP +
|
||||
POP_TOP +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
|
||||
op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
EXIT_IF(method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS));
|
||||
int total_args = oparg;
|
||||
_PyStackRef *arguments = args;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
|
|
@ -4732,18 +4756,26 @@ dummy_func(
|
|||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args == 0);
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
|
||||
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
|
||||
}
|
||||
|
||||
op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
PyMethodDef *meth = method->d_method;
|
||||
EXIT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS));
|
||||
PyTypeObject *d_type = method->d_common.d_type;
|
||||
|
||||
int total_args = oparg;
|
||||
_PyStackRef *arguments = args;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
arguments--;
|
||||
total_args++;
|
||||
}
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
|
||||
assert(self != NULL);
|
||||
EXIT_IF(!Py_IS_TYPE(self, d_type));
|
||||
STAT_INC(CALL, hit);
|
||||
PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal(
|
||||
callable,
|
||||
meth,
|
||||
method->d_method,
|
||||
self,
|
||||
arguments,
|
||||
total_args
|
||||
|
|
@ -4759,30 +4791,39 @@ dummy_func(
|
|||
_RECORD_CALLABLE +
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +
|
||||
_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) {
|
||||
assert(oparg == 0 || oparg == 1);
|
||||
op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
EXIT_IF(method->d_method->ml_flags != METH_NOARGS);
|
||||
int total_args = oparg;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
args--;
|
||||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args != 1);
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(
|
||||
PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null);
|
||||
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
|
||||
}
|
||||
|
||||
op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
PyMethodDef *meth = method->d_method;
|
||||
|
||||
assert(oparg == 1 || !PyStackRef_IsNull(self_or_null));
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
args--;
|
||||
}
|
||||
_PyStackRef self_stackref = args[0];
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref);
|
||||
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
|
||||
EXIT_IF(meth->ml_flags != METH_NOARGS);
|
||||
// CPython promises to check all non-vectorcall function calls.
|
||||
EXIT_IF(_Py_ReachedRecursionLimit(tstate));
|
||||
STAT_INC(CALL, hit);
|
||||
PyCFunction cfunc = meth->ml_meth;
|
||||
PyCFunction cfunc = method->d_method->ml_meth;
|
||||
PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL);
|
||||
_Py_LeaveRecursiveCallTstate(tstate);
|
||||
assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
|
||||
|
|
@ -4798,11 +4839,29 @@ dummy_func(
|
|||
_RECORD_CALLABLE +
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS +
|
||||
_CALL_METHOD_DESCRIPTOR_NOARGS +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
/* Builtin METH_FASTCALL methods, without keywords */
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
EXIT_IF(method->d_method->ml_flags != METH_FASTCALL);
|
||||
int total_args = oparg;
|
||||
if (!PyStackRef_IsNull(self_or_null)) {
|
||||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args == 0);
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(
|
||||
PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null);
|
||||
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
|
||||
}
|
||||
|
||||
op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) {
|
||||
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
|
||||
int total_args = oparg;
|
||||
_PyStackRef *arguments = args;
|
||||
|
|
@ -4810,19 +4869,12 @@ dummy_func(
|
|||
arguments--;
|
||||
total_args++;
|
||||
}
|
||||
EXIT_IF(total_args == 0);
|
||||
PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o;
|
||||
/* Builtin METH_FASTCALL methods, without keywords */
|
||||
EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type));
|
||||
PyMethodDef *meth = method->d_method;
|
||||
EXIT_IF(meth->ml_flags != METH_FASTCALL);
|
||||
PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]);
|
||||
assert(self != NULL);
|
||||
EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type));
|
||||
STAT_INC(CALL, hit);
|
||||
PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal(
|
||||
callable,
|
||||
meth,
|
||||
method->d_method,
|
||||
self,
|
||||
arguments,
|
||||
total_args
|
||||
|
|
@ -4837,6 +4889,7 @@ dummy_func(
|
|||
macro(CALL_METHOD_DESCRIPTOR_FAST) =
|
||||
unused/1 +
|
||||
unused/2 +
|
||||
_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST +
|
||||
_CALL_METHOD_DESCRIPTOR_FAST +
|
||||
_CHECK_PERIODIC_AT_END;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue