Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-01-06 07:22:09 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-132532: Add new DSL macros to better declare semantics of exits at ends of instructions/uops. (GH-137098)

Browse source
This commit is contained in:
Mark Shannon 2025-08-09 15:41:28 +01:00 • committed by GitHub
parent c744b57fd6
commit af15e1d13e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 524 additions and 609 deletions
Download patch file Download diff file Expand all files Collapse all files

74
Python/bytecodes.c
View file

@ -153,22 +153,19 @@ dummy_func(
macro(NOT_TAKEN) = NOP;
op(_CHECK_PERIODIC, (--)) {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
int err = _Py_HandlePending(tstate);
ERROR_IF(err != 0);
}
int err = check_periodics(tstate);
ERROR_IF(err != 0);
}
replaced op(_CHECK_PERIODIC_AT_END, (--)) {
int err = check_periodics(tstate);
ERROR_IF(err != 0);
}
op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) {
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
int err = _Py_HandlePending(tstate);
ERROR_IF(err != 0);
}
int err = check_periodics(tstate);
ERROR_IF(err != 0);
}
}
@ -3794,8 +3791,8 @@ dummy_func(
ERROR_IF(err);
}
macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC;
macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC;
macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END;
macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END;
op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
@ -3916,7 +3913,7 @@ dummy_func(
unused/2 +
_CHECK_IS_NOT_PY_CALLABLE +
_CALL_NON_PY_GENERAL +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) {
EXIT_IF(!PyStackRef_IsNull(null));
@ -4073,7 +4070,7 @@ dummy_func(
_GUARD_NOS_NULL +
_GUARD_CALLABLE_STR_1 +
_CALL_STR_1 +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
@ -4101,7 +4098,7 @@ dummy_func(
_GUARD_NOS_NULL +
_GUARD_CALLABLE_TUPLE_1 +
_CALL_TUPLE_1 +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
@ -4197,7 +4194,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_BUILTIN_CLASS +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_O functions */
@ -4232,7 +4229,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_BUILTIN_O +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_FASTCALL functions, without keywords */
@ -4269,7 +4266,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_BUILTIN_FAST +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
@ -4305,7 +4302,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_BUILTIN_FAST_WITH_KEYWORDS +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
macro(CALL_LEN) =
unused/1 +
@ -4443,7 +4440,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_O +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
@ -4485,7 +4482,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) {
assert(oparg == 0 || oparg == 1);
@ -4523,7 +4520,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_NOARGS +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
@ -4564,7 +4561,7 @@ dummy_func(
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_FAST +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
// Cache layout: counter/1, func_version/2
family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = {
@ -4820,7 +4817,7 @@ dummy_func(
unused/2 +
_CHECK_IS_NOT_PY_CALLABLE_KW +
_CALL_KW_NON_PY +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs -- func, unused, callargs, kwargs)) {
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
@ -4921,12 +4918,12 @@ dummy_func(
macro(CALL_FUNCTION_EX) =
_MAKE_CALLARGS_A_TUPLE +
_DO_CALL_FUNCTION_EX +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
macro(INSTRUMENTED_CALL_FUNCTION_EX) =
_MAKE_CALLARGS_A_TUPLE +
_DO_CALL_FUNCTION_EX +
_CHECK_PERIODIC;
_CHECK_PERIODIC_AT_END;
inst(MAKE_FUNCTION, (codeobj_st -- func)) {
PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st);
@ -5189,23 +5186,20 @@ dummy_func(
op (_GUARD_IS_TRUE_POP, (flag -- )) {
int is_true = PyStackRef_IsTrue(flag);
DEAD(flag);
SYNC_SP();
EXIT_IF(!is_true);
AT_END_EXIT_IF(!is_true);
}
op (_GUARD_IS_FALSE_POP, (flag -- )) {
int is_false = PyStackRef_IsFalse(flag);
DEAD(flag);
SYNC_SP();
EXIT_IF(!is_false);
AT_END_EXIT_IF(!is_false);
}
op (_GUARD_IS_NONE_POP, (val -- )) {
int is_none = PyStackRef_IsNone(val);
if (!is_none) {
PyStackRef_CLOSE(val);
SYNC_SP();
EXIT_IF(1);
AT_END_EXIT_IF(1);
}
DEAD(val);
}
@ -5213,8 +5207,7 @@ dummy_func(
op (_GUARD_IS_NOT_NONE_POP, (val -- )) {
int is_none = PyStackRef_IsNone(val);
PyStackRef_CLOSE(val);
SYNC_SP();
EXIT_IF(is_none);
AT_END_EXIT_IF(is_none);
}
op(_JUMP_TO_TOP, (--)) {
@ -5380,6 +5373,11 @@ dummy_func(
GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET());
}
tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) {
int err = _Py_HandlePending(tstate);
GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET());
}
tier2 op(_ERROR_POP_N, (target/2 --)) {
assert(oparg == 0);
frame->instr_ptr = _PyFrame_GetBytecode(frame) + target;
@ -5391,11 +5389,11 @@ dummy_func(
* ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */
tier2 op(_TIER2_RESUME_CHECK, (--)) {
#if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0);
HANDLE_PENDING_AND_DEOPT_IF(_Py_emscripten_signal_clock == 0);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK);
HANDLE_PENDING_AND_DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK);
assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version));
}

1
Python/ceval.c
View file

@ -1211,6 +1211,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
printf(" @ %d -> %s]\n",
(int)(next_uop - current_executor->trace - 1),
_PyOpcode_OpName[frame->instr_ptr->op.code]);
fflush(stdout);
}
#endif
assert(next_uop[-1].format == UOP_FORMAT_JUMP);

11
Python/ceval_macros.h
View file

@ -422,3 +422,14 @@ do { \
_PyObjectArray_Free(NAME - 1, NAME##_temp);
#define CONVERSION_FAILED(NAME) ((NAME) == NULL)
static inline int
check_periodics(PyThreadState *tstate) {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
return _Py_HandlePending(tstate);
}
return 0;
}

38
Python/executor_cases.c.h generated
View file

@ -13,31 +13,25 @@
}
case _CHECK_PERIODIC: {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_ERROR();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_ERROR();
}
break;
}
/* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */
case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: {
oparg = CURRENT_OPARG();
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_ERROR();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_ERROR();
}
}
break;
@ -7443,6 +7437,14 @@
break;
}
case _HANDLE_PENDING_AND_DEOPT: {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET());
break;
}
case _ERROR_POP_N: {
oparg = CURRENT_OPARG();
uint32_t target = (uint32_t)CURRENT_OPERAND0();

500
Python/generated_cases.c.h generated
View file

@ -1670,26 +1670,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -2175,26 +2167,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -2292,26 +2276,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -2409,26 +2385,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -2501,26 +2469,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -2676,26 +2636,18 @@
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = result;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = result;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = result;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -3268,26 +3220,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -3659,26 +3603,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -3790,26 +3726,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -3891,26 +3819,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -4005,26 +3925,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -4128,26 +4040,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -4421,26 +4325,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -4501,26 +4397,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6517,26 +6405,18 @@
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -6692,26 +6572,18 @@
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
// _CHECK_PERIODIC
// _CHECK_PERIODIC_AT_END
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
stack_pointer[0] = result;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
stack_pointer[0] = result;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
stack_pointer[0] = result;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -7093,15 +6965,11 @@
/* Skip 1 cache entry */
// _CHECK_PERIODIC
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
// _MONITOR_JUMP_BACKWARD
@ -7484,15 +7352,11 @@
// _CHECK_PERIODIC_IF_NOT_YIELD_FROM
{
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
}
@ -7721,15 +7585,11 @@
}
// _CHECK_PERIODIC
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
// _JUMP_BACKWARD_NO_INTERRUPT
@ -7754,15 +7614,11 @@
/* Skip 1 cache entry */
// _CHECK_PERIODIC
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
// _JUMP_BACKWARD_NO_INTERRUPT
@ -7833,15 +7689,11 @@
/* Skip 1 cache entry */
// _CHECK_PERIODIC
{
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
// _JUMP_BACKWARD_NO_INTERRUPT
@ -10419,15 +10271,11 @@
// _CHECK_PERIODIC_IF_NOT_YIELD_FROM
{
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
_Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
QSBR_QUIESCENT_STATE(tstate);
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_HandlePending(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
}

21
Python/optimizer.c
View file

@ -432,6 +432,7 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = {
[_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE,
[_FOR_ITER] = _FOR_ITER_TIER_TWO,
[_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO,
[_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK,
};
static const uint8_t
@ -776,9 +777,12 @@ translate_bytecode_to_trace(
case OPARG_REPLACED:
uop = _PyUOp_Replacements[uop];
assert(uop != 0);
uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255);
if (uop == _TIER2_RESUME_CHECK) {
target = next_inst;
}
#ifdef Py_DEBUG
{
uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255);
else {
uint32_t jump_target = next_inst + oparg;
assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR);
assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER);
@ -1043,9 +1047,15 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length)
_PyUOpInstruction *inst = &buffer[i];
int opcode = inst->opcode;
int32_t target = (int32_t)uop_get_target(inst);
if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) {
uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ?
_EXIT_TRACE : _DEOPT;
uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG);
if (exit_flags) {
uint16_t exit_op = _EXIT_TRACE;
if (exit_flags & HAS_DEOPT_FLAG) {
exit_op = _DEOPT;
}
else if (exit_flags & HAS_PERIODIC_FLAG) {
exit_op = _HANDLE_PENDING_AND_DEOPT;
}
int32_t jump_target = target;
if (is_for_iter_test[opcode]) {
/* Target the POP_TOP immediately after the END_FOR,
@ -1159,6 +1169,7 @@ sanity_check(_PyExecutorObject *executor)
uint16_t opcode = inst->opcode;
CHECK(
opcode == _DEOPT ||
opcode == _HANDLE_PENDING_AND_DEOPT ||
opcode == _EXIT_TRACE ||
opcode == _ERROR_POP_N);
}

6
Python/optimizer_cases.c.h generated
View file

@ -11,6 +11,8 @@
break;
}
/* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 */
case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: {
break;
}
@ -3198,6 +3200,10 @@
break;
}
case _HANDLE_PENDING_AND_DEOPT: {
break;
}
case _ERROR_POP_N: {
break;
}