Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-05-03 00:58:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-144179: Add value recording to JIT tracing front-end (GH-144303)

Browse source
This commit is contained in:
Mark Shannon 2026-02-02 16:57:04 +00:00 • committed by GitHub
parent 5f91577cdd
commit 141fd8b894
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 1702 additions and 1274 deletions
Download patch file Download diff file Expand all files Collapse all files

132
Python/bytecodes.c
View file

@ -568,6 +568,7 @@ dummy_func(
macro(TO_BOOL_ALWAYS_TRUE) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_REPLACE_WITH_TRUE +
POP_TOP;
@ -1107,6 +1108,7 @@ dummy_func(
}
macro(BINARY_OP_SUBSCR_GETITEM) =
_RECORD_TOS +
unused/5 + // Skip over the counter and cache
_CHECK_PEP_523 +
_BINARY_OP_SUBSCR_CHECK_FUNC +
@ -1267,7 +1269,7 @@ dummy_func(
// The stack effect here is a bit misleading.
// retval is popped from the stack, but res
// is pushed to a different frame, the callers' frame.
inst(RETURN_VALUE, (retval -- res)) {
op(_RETURN_VALUE, (retval -- res)) {
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
_PyStackRef temp = PyStackRef_MakeHeapSafe(retval);
DEAD(retval);
@ -1291,9 +1293,13 @@ dummy_func(
ERROR_IF(err);
}
macro(RETURN_VALUE) =
_RECORD_CALLER_CODE +
_RETURN_VALUE;
macro(INSTRUMENTED_RETURN_VALUE) =
_RETURN_VALUE_EVENT +
RETURN_VALUE;
_RETURN_VALUE;
inst(GET_AITER, (obj -- iter)) {
unaryfunc getter = NULL;
@ -1434,7 +1440,7 @@ dummy_func(
_SEND_GEN_FRAME +
_PUSH_FRAME;
inst(YIELD_VALUE, (retval -- value)) {
op(_YIELD_VALUE, (retval -- value)) {
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
// or throw() call.
@ -1470,6 +1476,10 @@ dummy_func(
LLTRACE_RESUME_FRAME();
}
macro(YIELD_VALUE) =
_RECORD_CALLER_CODE +
_YIELD_VALUE;
tier1 op(_YIELD_VALUE_EVENT, (val -- val)) {
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
@ -1485,7 +1495,7 @@ dummy_func(
macro(INSTRUMENTED_YIELD_VALUE) =
_YIELD_VALUE_EVENT +
YIELD_VALUE;
_YIELD_VALUE;
inst(POP_EXCEPT, (exc_value -- )) {
_PyErr_StackItem *exc_info = tstate->exc_info;
@ -2445,6 +2455,7 @@ dummy_func(
macro(LOAD_ATTR_INSTANCE_VALUE) =
unused/1 + // Skip over the counter
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_CHECK_MANAGED_OBJECT_HAS_VALUES +
_LOAD_ATTR_INSTANCE_VALUE +
@ -2526,6 +2537,7 @@ dummy_func(
macro(LOAD_ATTR_WITH_HINT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_LOAD_ATTR_WITH_HINT +
POP_TOP +
@ -2551,6 +2563,7 @@ dummy_func(
macro(LOAD_ATTR_SLOT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_LOAD_ATTR_SLOT + // NOTE: This action may also deopt
POP_TOP +
@ -2581,8 +2594,9 @@ dummy_func(
macro(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) =
unused/1 +
_CHECK_ATTR_CLASS +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_CHECK_ATTR_CLASS +
_LOAD_ATTR_CLASS +
_PUSH_NULL_CONDITIONAL;
@ -2604,8 +2618,9 @@ dummy_func(
macro(LOAD_ATTR_PROPERTY) =
unused/1 +
_CHECK_PEP_523 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_CHECK_PEP_523 +
unused/2 +
_LOAD_ATTR_PROPERTY_FRAME +
_SAVE_RETURN_OFFSET +
@ -2715,6 +2730,7 @@ dummy_func(
macro(STORE_ATTR_WITH_HINT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_STORE_ATTR_WITH_HINT +
POP_TOP;
@ -2735,6 +2751,7 @@ dummy_func(
macro(STORE_ATTR_SLOT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_STORE_ATTR_SLOT +
POP_TOP;
@ -3021,7 +3038,7 @@ dummy_func(
oparg >>= 8;
insert_exec_at--;
}
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg, NULL);
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, stack_pointer, 0, NULL, oparg, NULL);
if (succ) {
ENTER_TRACING();
}
@ -3506,6 +3523,7 @@ dummy_func(
}
macro(FOR_ITER_GEN) =
_RECORD_NOS +
unused/1 +
_CHECK_PEP_523 +
_FOR_ITER_GEN_FRAME +
@ -3640,6 +3658,7 @@ dummy_func(
macro(LOAD_ATTR_METHOD_WITH_VALUES) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT +
_GUARD_KEYS_VERSION +
@ -3658,6 +3677,7 @@ dummy_func(
macro(LOAD_ATTR_METHOD_NO_DICT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
unused/2 +
_LOAD_ATTR_METHOD_NO_DICT;
@ -3672,6 +3692,7 @@ dummy_func(
macro(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT +
_GUARD_KEYS_VERSION +
@ -3688,6 +3709,7 @@ dummy_func(
macro(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
unused/2 +
_LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
@ -3711,6 +3733,7 @@ dummy_func(
macro(LOAD_ATTR_METHOD_LAZY_DICT) =
unused/1 +
_RECORD_TOS_TYPE +
_GUARD_TYPE_VERSION +
_CHECK_ATTR_METHOD_LAZY_DICT +
unused/1 +
@ -3879,6 +3902,7 @@ dummy_func(
}
macro(CALL_PY_GENERAL) =
_RECORD_CALLABLE +
unused/1 + // Skip over the counter
_CHECK_PEP_523 +
_CHECK_FUNCTION_VERSION +
@ -3909,6 +3933,7 @@ dummy_func(
}
macro(CALL_BOUND_METHOD_GENERAL) =
_RECORD_BOUND_METHOD +
unused/1 + // Skip over the counter
_CHECK_PEP_523 +
_CHECK_METHOD_VERSION +
@ -3948,6 +3973,7 @@ dummy_func(
}
macro(CALL_NON_PY_GENERAL) =
_RECORD_CALLABLE +
unused/1 + // Skip over the counter
unused/2 +
_CHECK_IS_NOT_PY_CALLABLE +
@ -4021,6 +4047,7 @@ dummy_func(
}
macro(CALL_BOUND_METHOD_EXACT_ARGS) =
_RECORD_BOUND_METHOD +
unused/1 + // Skip over the counter
_CHECK_PEP_523 +
_CHECK_CALL_BOUND_METHOD_EXACT_ARGS +
@ -4035,6 +4062,7 @@ dummy_func(
_PUSH_FRAME;
macro(CALL_PY_EXACT_ARGS) =
_RECORD_CALLABLE +
unused/1 + // Skip over the counter
_CHECK_PEP_523 +
_CHECK_FUNCTION_VERSION +
@ -4187,6 +4215,7 @@ dummy_func(
}
macro(CALL_ALLOC_AND_ENTER_INIT) =
_RECORD_CALLABLE +
unused/1 +
_CHECK_PEP_523 +
_CHECK_AND_ALLOCATE_OBJECT +
@ -4227,6 +4256,7 @@ dummy_func(
}
macro(CALL_BUILTIN_CLASS) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_BUILTIN_CLASS +
@ -4262,6 +4292,7 @@ dummy_func(
}
macro(CALL_BUILTIN_O) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_BUILTIN_O +
@ -4294,6 +4325,7 @@ dummy_func(
}
macro(CALL_BUILTIN_FAST) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_BUILTIN_FAST +
@ -4320,6 +4352,7 @@ dummy_func(
}
macro(CALL_BUILTIN_FAST_WITH_KEYWORDS) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_BUILTIN_FAST_WITH_KEYWORDS +
@ -4463,6 +4496,7 @@ dummy_func(
}
macro(CALL_METHOD_DESCRIPTOR_O) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_O +
@ -4505,6 +4539,7 @@ dummy_func(
}
macro(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS +
@ -4543,6 +4578,7 @@ dummy_func(
}
macro(CALL_METHOD_DESCRIPTOR_NOARGS) =
_RECORD_CALLABLE +
unused/1 +
unused/2 +
_CALL_METHOD_DESCRIPTOR_NOARGS +
@ -4968,6 +5004,7 @@ dummy_func(
}
macro(CALL_EX_PY) =
_RECORD_4OS +
unused/1 +
_CHECK_PEP_523 +
_MAKE_CALLARGS_A_TUPLE +
@ -5037,7 +5074,7 @@ dummy_func(
*ptr = attr;
}
inst(RETURN_GENERATOR, (-- res)) {
op(_RETURN_GENERATOR, (-- res)) {
assert(PyStackRef_FunctionCheck(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
@ -5060,6 +5097,10 @@ dummy_func(
LLTRACE_RESUME_FRAME();
}
macro(RETURN_GENERATOR) =
_RECORD_CALLER_CODE +
_RETURN_GENERATOR;
inst(BUILD_SLICE, (args[oparg] -- slice)) {
PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]);
@ -5588,7 +5629,7 @@ dummy_func(
// Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG.
// The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG.
// So setting it to anything else is wrong.
int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg, previous_executor);
int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, stack_pointer, chain_depth, exit, target->op.arg, previous_executor);
exit->temperature = restart_backoff_counter(exit->temperature);
if (succ) {
GOTO_TIER_ONE_CONTINUE_TRACING(target);
@ -5607,37 +5648,77 @@ dummy_func(
}
tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) {
_Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(_PUSH_FRAME);
_Py_CODEUNIT *target = frame->instr_ptr;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += IP_OFFSET_OF(_PUSH_FRAME);
EXIT_IF(true);
}
}
tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) {
_Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(YIELD_VALUE);
_Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += IP_OFFSET_OF(YIELD_VALUE);
frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND;
EXIT_IF(true);
}
}
tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) {
_Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_VALUE);
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += IP_OFFSET_OF(RETURN_VALUE);
frame->instr_ptr += frame->return_offset;
EXIT_IF(true);
}
}
tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) {
_Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_GENERATOR);
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += IP_OFFSET_OF(RETURN_GENERATOR);
frame->instr_ptr += frame->return_offset;
EXIT_IF(true);
}
}
/* Record ops for jit tracer.
*
* NOTE: These uops are NOPs for normal evaluation.
* They are only executed during trace recording */
tier2 op(_RECORD_TOS, (tos -- tos)) {
RECORD_VALUE(PyStackRef_AsPyObjectBorrow(tos));
}
tier2 op(_RECORD_TOS_TYPE, (tos -- tos)) {
RECORD_VALUE(Py_TYPE(PyStackRef_AsPyObjectBorrow(tos)));
}
tier2 op(_RECORD_NOS, (nos, tos -- nos, tos)) {
RECORD_VALUE(PyStackRef_AsPyObjectBorrow(nos));
}
tier2 op(_RECORD_4OS, (value, _3os, nos, tos -- value, _3os, nos, tos)) {
RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value));
}
tier2 op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) {
RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func));
}
tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (Py_TYPE(callable_o) == &PyMethod_Type) {
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
RECORD_VALUE(func);
}
}
tier2 op(_RECORD_CALLER_CODE, ( -- )) {
_PyInterpreterFrame *caller_frame = frame->previous;
if (caller_frame->owner < FRAME_OWNED_BY_INTERPRETER) {
PyCodeObject *code = _PyFrame_GetCode(frame->previous);
RECORD_VALUE(code);
}
}
label(pop_2_error) {
stack_pointer -= 2;
assert(WITHIN_STACK_BOUNDS());
@ -5802,6 +5883,9 @@ dummy_func(
opcode == INTERPRETER_EXIT ||
(opcode >= MIN_INSTRUMENTED_OPCODE && opcode != ENTER_EXECUTOR)
);
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
_PyJitTracerState *tracer = _tstate->jit_tracer_state;
assert(tracer != NULL);
int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0);
if (full) {
LEAVE_TRACING();
@ -5809,13 +5893,7 @@ dummy_func(
ERROR_IF(err < 0);
DISPATCH();
}
// Super instructions. Instruction deopted. There's a mismatch in what the stack expects
// in the optimizer. So we have to reflect in the trace correctly.
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
// JIT should have disabled super instructions, as we can
// do these optimizations ourselves in the JIT.
_PyJitTracerState *tracer = _tstate->jit_tracer_state;
assert(tracer != NULL);
Py_CLEAR(tracer->prev_state.recorded_value);
tracer->prev_state.instr = next_instr;
PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) {
@ -5828,6 +5906,12 @@ dummy_func(
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
(&next_instr[1])->counter = trigger_backoff_counter();
}
uint8_t record_func_index = _PyOpcode_RecordFunctionIndices[opcode];
if (record_func_index) {
_Py_RecordFuncPtr doesnt_escape = _PyOpcode_RecordFunctions[record_func_index];
doesnt_escape(frame, stack_pointer, oparg, &tracer->prev_state.recorded_value);
}
DISPATCH_GOTO_NON_TRACING();
#else
(void)prev_instr;

94
Python/executor_cases.c.h generated
View file

@ -20150,7 +20150,7 @@
_PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit);
assert(tstate->current_executor == (PyObject *)previous_executor);
int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow;
int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg, previous_executor);
int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, stack_pointer, chain_depth, exit, target->op.arg, previous_executor);
exit->temperature = restart_backoff_counter(exit->temperature);
if (succ) {
GOTO_TIER_ONE_CONTINUE_TRACING(target);
@ -20171,11 +20171,9 @@
case _GUARD_IP__PUSH_FRAME_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
_Py_CODEUNIT *target = frame->instr_ptr;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
if (true) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
@ -20183,7 +20181,6 @@
}
}
SET_CURRENT_CACHED_VALUES(0);
#undef OFFSET_OF__PUSH_FRAME
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20192,11 +20189,9 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
_Py_CODEUNIT *target = frame->instr_ptr;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
@ -20206,7 +20201,6 @@
}
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
#undef OFFSET_OF__PUSH_FRAME
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20216,11 +20210,9 @@
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
_Py_CODEUNIT *target = frame->instr_ptr;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
@ -20232,7 +20224,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
#undef OFFSET_OF__PUSH_FRAME
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20243,11 +20234,9 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
_Py_CODEUNIT *target = frame->instr_ptr;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
@ -20261,7 +20250,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
#undef OFFSET_OF__PUSH_FRAME
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20269,11 +20257,10 @@
case _GUARD_IP_YIELD_VALUE_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND;
if (true) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
@ -20281,7 +20268,6 @@
}
}
SET_CURRENT_CACHED_VALUES(0);
#undef OFFSET_OF_YIELD_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20290,11 +20276,10 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
@ -20304,7 +20289,6 @@
}
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
#undef OFFSET_OF_YIELD_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20314,11 +20298,10 @@
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
@ -20330,7 +20313,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
#undef OFFSET_OF_YIELD_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20341,11 +20323,10 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
@ -20359,7 +20340,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
#undef OFFSET_OF_YIELD_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20367,11 +20347,10 @@
case _GUARD_IP_RETURN_VALUE_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
@ -20379,7 +20358,6 @@
}
}
SET_CURRENT_CACHED_VALUES(0);
#undef OFFSET_OF_RETURN_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20388,11 +20366,10 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
@ -20402,7 +20379,6 @@
}
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
#undef OFFSET_OF_RETURN_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20412,11 +20388,10 @@
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
@ -20428,7 +20403,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
#undef OFFSET_OF_RETURN_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20439,11 +20413,10 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
@ -20457,7 +20430,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
#undef OFFSET_OF_RETURN_VALUE
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20465,11 +20437,10 @@
case _GUARD_IP_RETURN_GENERATOR_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
@ -20477,7 +20448,6 @@
}
}
SET_CURRENT_CACHED_VALUES(0);
#undef OFFSET_OF_RETURN_GENERATOR
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20486,11 +20456,10 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
@ -20500,7 +20469,6 @@
}
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
#undef OFFSET_OF_RETURN_GENERATOR
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20510,11 +20478,10 @@
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
@ -20526,7 +20493,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
#undef OFFSET_OF_RETURN_GENERATOR
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
@ -20537,11 +20503,10 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
_Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
frame->instr_ptr += frame->return_offset;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
@ -20555,7 +20520,6 @@
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
#undef OFFSET_OF_RETURN_GENERATOR
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}

193
Python/generated_cases.c.h generated
View file

@ -7723,7 +7723,7 @@
oparg >>= 8;
insert_exec_at--;
}
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg, NULL);
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, stack_pointer, 0, NULL, oparg, NULL);
if (succ) {
ENTER_TRACING();
}
@ -8017,10 +8017,21 @@
_PyStackRef attr;
_PyStackRef *null;
/* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _CHECK_ATTR_CLASS
{
uint32_t type_version = read_u32(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!PyType_Check(owner_o)) {
UPDATE_MISS_STATS(LOAD_ATTR);
@ -8034,17 +8045,6 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _GUARD_TYPE_VERSION
{
uint32_t type_version = read_u32(&this_instr[4].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _LOAD_ATTR_CLASS
{
PyObject *descr = read_obj(&this_instr[6].cache);
@ -8594,14 +8594,6 @@
_PyStackRef owner;
_PyStackRef new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
if (IS_PEP523_HOOKED(tstate)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _GUARD_TYPE_VERSION
{
owner = stack_pointer[-1];
@ -8614,6 +8606,14 @@
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
// _CHECK_PEP_523
{
if (IS_PEP523_HOOKED(tstate)) {
UPDATE_MISS_STATS(LOAD_ATTR);
assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR));
JUMP_TO_PREDICTED(LOAD_ATTR);
}
}
/* Skip 2 cache entries */
// _LOAD_ATTR_PROPERTY_FRAME
{
@ -10446,30 +10446,33 @@
next_instr += 1;
INSTRUCTION_STATS(RETURN_GENERATOR);
_PyStackRef res;
assert(PyStackRef_FunctionCheck(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (gen == NULL) {
JUMP_TO_LABEL(error);
// _RETURN_GENERATOR
{
assert(PyStackRef_FunctionCheck(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (gen == NULL) {
JUMP_TO_LABEL(error);
}
assert(STACK_LEVEL() <= 2);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
frame->instr_ptr++;
_PyFrame_Copy(frame, gen_frame);
assert(frame->frame_obj == NULL);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
LOAD_IP(frame->return_offset);
stack_pointer = _PyFrame_GetStackPointer(frame);
res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen);
LLTRACE_RESUME_FRAME();
}
assert(STACK_LEVEL() <= 2);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *gen_frame = &gen->gi_iframe;
frame->instr_ptr++;
_PyFrame_Copy(frame, gen_frame);
assert(frame->frame_obj == NULL);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
LOAD_IP(frame->return_offset);
stack_pointer = _PyFrame_GetStackPointer(frame);
res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen);
LLTRACE_RESUME_FRAME();
stack_pointer[0] = res;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@ -10486,21 +10489,24 @@
INSTRUCTION_STATS(RETURN_VALUE);
_PyStackRef retval;
_PyStackRef res;
retval = stack_pointer[-1];
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
_PyStackRef temp = PyStackRef_MakeHeapSafe(retval);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(STACK_LEVEL() == 0);
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
stack_pointer = _PyFrame_GetStackPointer(frame);
LOAD_IP(frame->return_offset);
res = temp;
LLTRACE_RESUME_FRAME();
// _RETURN_VALUE
{
retval = stack_pointer[-1];
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
_PyStackRef temp = PyStackRef_MakeHeapSafe(retval);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(STACK_LEVEL() == 0);
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
stack_pointer = _PyFrame_GetStackPointer(frame);
LOAD_IP(frame->return_offset);
res = temp;
LLTRACE_RESUME_FRAME();
}
stack_pointer[0] = res;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@ -11889,6 +11895,9 @@
opcode == INTERPRETER_EXIT ||
(opcode >= MIN_INSTRUMENTED_OPCODE && opcode != ENTER_EXECUTOR)
);
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
_PyJitTracerState *tracer = _tstate->jit_tracer_state;
assert(tracer != NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0);
stack_pointer = _PyFrame_GetStackPointer(frame);
@ -11902,9 +11911,9 @@
}
DISPATCH();
}
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
_PyJitTracerState *tracer = _tstate->jit_tracer_state;
assert(tracer != NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_CLEAR(tracer->prev_state.recorded_value);
stack_pointer = _PyFrame_GetStackPointer(frame);
tracer->prev_state.instr = next_instr;
PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable);
if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) {
@ -11918,6 +11927,11 @@
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
(&next_instr[1])->counter = trigger_backoff_counter();
}
uint8_t record_func_index = _PyOpcode_RecordFunctionIndices[opcode];
if (record_func_index) {
_Py_RecordFuncPtr doesnt_escape = _PyOpcode_RecordFunctions[record_func_index];
doesnt_escape(frame, stack_pointer, oparg, &tracer->prev_state.recorded_value);
}
DISPATCH_GOTO_NON_TRACING();
#else
(void)prev_instr;
@ -12309,37 +12323,40 @@
INSTRUCTION_STATS(YIELD_VALUE);
_PyStackRef retval;
_PyStackRef value;
retval = stack_pointer[-1];
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
frame->instr_ptr++;
PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
_PyStackRef temp = retval;
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD;
FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg);
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
#if TIER_ONE
assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE ||
// _YIELD_VALUE
{
retval = stack_pointer[-1];
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
frame->instr_ptr++;
PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
_PyStackRef temp = retval;
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD;
FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg);
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
#if TIER_ONE
assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE ||
frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR);
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
value = PyStackRef_MakeHeapSafe(temp);
LLTRACE_RESUME_FRAME();
#endif
stack_pointer = _PyFrame_GetStackPointer(frame);
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
value = PyStackRef_MakeHeapSafe(temp);
LLTRACE_RESUME_FRAME();
}
stack_pointer[0] = value;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);

24
Python/optimizer.c
View file

@ -955,6 +955,11 @@ _PyJit_translate_single_bytecode_to_trace(
assert(next->op.code == STORE_FAST);
operand = next->op.arg;
}
else if (_PyUop_Flags[uop] & HAS_RECORDS_VALUE_FLAG) {
PyObject *recorded_value = tracer->prev_state.recorded_value;
tracer->prev_state.recorded_value = NULL;
operand = (uintptr_t)recorded_value;
}
// All other instructions
ADD_TO_TRACE(uop, oparg, operand, target);
}
@ -997,7 +1002,7 @@ _PyJit_translate_single_bytecode_to_trace(
Py_NO_INLINE int
_PyJit_TryInitializeTracing(
PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr,
_Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth,
_Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, _PyStackRef *stack_pointer, int chain_depth,
_PyExitData *exit, int oparg, _PyExecutorObject *current_executor)
{
_PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
@ -1048,14 +1053,20 @@ _PyJit_TryInitializeTracing(
tracer->initial_state.func = (PyFunctionObject *)Py_NewRef(func);
tracer->initial_state.executor = (_PyExecutorObject *)Py_XNewRef(current_executor);
tracer->initial_state.exit = exit;
tracer->initial_state.stack_depth = curr_stackdepth;
tracer->initial_state.stack_depth = stack_pointer - _PyFrame_Stackbase(frame);
tracer->initial_state.chain_depth = chain_depth;
tracer->prev_state.dependencies_still_valid = true;
tracer->prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame));
tracer->prev_state.instr = curr_instr;
tracer->prev_state.instr_frame = frame;
tracer->prev_state.instr_oparg = oparg;
tracer->prev_state.instr_stacklevel = curr_stackdepth;
tracer->prev_state.instr_stacklevel = tracer->initial_state.stack_depth;
tracer->prev_state.recorded_value = NULL;
uint8_t record_func_index = _PyOpcode_RecordFunctionIndices[curr_instr->op.code];
if (record_func_index) {
_Py_RecordFuncPtr record_func = _PyOpcode_RecordFunctions[record_func_index];
record_func(frame, stack_pointer, oparg, &tracer->prev_state.recorded_value);
}
assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL));
tracer->initial_state.jump_backward_instr = curr_instr;
@ -1099,6 +1110,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate, int err)
Py_CLEAR(tracer->initial_state.func);
Py_CLEAR(tracer->initial_state.executor);
Py_CLEAR(tracer->prev_state.instr_code);
Py_CLEAR(tracer->prev_state.recorded_value);
uop_buffer_init(&tracer->code_buffer, &tracer->uop_array[0], UOP_MAX_TRACE_LENGTH);
tracer->is_tracing = false;
}
@ -1526,6 +1538,10 @@ uop_optimize(
buffer[pc].opcode = opcode + oparg + 1 - _PyUop_Replication[opcode].start;
assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0);
}
else if (_PyUop_Flags[opcode] & HAS_RECORDS_VALUE_FLAG) {
Py_XDECREF((PyObject *)(uintptr_t)buffer[pc].operand0);
buffer[pc].opcode = _NOP;
}
else if (is_terminator(&buffer[pc])) {
break;
}
@ -1926,6 +1942,8 @@ _Py_Executors_InvalidateCold(PyInterpreterState *interp)
_Py_Executors_InvalidateAll(interp, 0);
}
#include "record_functions.c.h"
static int
escape_angles(const char *input, Py_ssize_t size, char *buffer) {
int written = 0;

1
Python/optimizer_bytecodes.c
View file

@ -177,6 +177,7 @@ dummy_func(void) {
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
assert(type_version);
assert(this_instr[-1].opcode == _RECORD_TOS_TYPE);
if (sym_matches_type_version(owner, type_version)) {
ADD_OP(_NOP, 0, 0);
} else {

29
Python/optimizer_cases.c.h generated
View file

@ -1833,6 +1833,7 @@
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)this_instr->operand0;
assert(type_version);
assert(this_instr[-1].opcode == _RECORD_TOS_TYPE);
if (sym_matches_type_version(owner, type_version)) {
ADD_OP(_NOP, 0, 0);
} else {
@ -4176,5 +4177,33 @@
break;
}
case _RECORD_TOS: {
break;
}
case _RECORD_TOS_TYPE: {
break;
}
case _RECORD_NOS: {
break;
}
case _RECORD_4OS: {
break;
}
case _RECORD_CALLABLE: {
break;
}
case _RECORD_BOUND_METHOD: {
break;
}
case _RECORD_CALLER_CODE: {
break;
}
/* _TRACE_RECORD is not a viable micro-op for tier 2 */

115
Python/record_functions.c.h generated Normal file
View file

@ -0,0 +1,115 @@
// This file is generated by Tools/cases_generator/record_function_generator.py
// from:
// Python/bytecodes.c
// Do not edit!
#ifdef TIER_ONE
#error "This file is for Tier 2 only"
#endif
void _PyOpcode_RecordFunction_TOS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyStackRef tos;
tos = stack_pointer[-1];
*recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(tos);
Py_INCREF(*recorded_value);
}
void _PyOpcode_RecordFunction_TOS_TYPE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyStackRef tos;
tos = stack_pointer[-1];
*recorded_value = (PyObject *)Py_TYPE(PyStackRef_AsPyObjectBorrow(tos));
Py_INCREF(*recorded_value);
}
void _PyOpcode_RecordFunction_NOS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyStackRef nos;
nos = stack_pointer[-2];
*recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(nos);
Py_INCREF(*recorded_value);
}
void _PyOpcode_RecordFunction_4OS(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyStackRef value;
value = stack_pointer[-4];
*recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(value);
Py_INCREF(*recorded_value);
}
void _PyOpcode_RecordFunction_CALLABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyStackRef func;
func = stack_pointer[-2 - oparg];
*recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func);
Py_INCREF(*recorded_value);
}
void _PyOpcode_RecordFunction_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyStackRef callable;
callable = stack_pointer[-2 - oparg];
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (Py_TYPE(callable_o) == &PyMethod_Type) {
PyObject *func = ((PyMethodObject *)callable_o)->im_func;
*recorded_value = (PyObject *)func;
Py_INCREF(*recorded_value);
}
}
void _PyOpcode_RecordFunction_CALLER_CODE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) {
_PyInterpreterFrame *caller_frame = frame->previous;
if (caller_frame->owner < FRAME_OWNED_BY_INTERPRETER) {
PyCodeObject *code = _PyFrame_GetCode(frame->previous);
*recorded_value = (PyObject *)code;
Py_INCREF(*recorded_value);
}
}
#define _RECORD_TOS_TYPE_INDEX 1
#define _RECORD_TOS_INDEX 2
#define _RECORD_CALLER_CODE_INDEX 3
#define _RECORD_NOS_INDEX 4
#define _RECORD_CALLABLE_INDEX 5
#define _RECORD_BOUND_METHOD_INDEX 6
#define _RECORD_4OS_INDEX 7
const uint8_t _PyOpcode_RecordFunctionIndices[256] = {
[TO_BOOL_ALWAYS_TRUE] = _RECORD_TOS_TYPE_INDEX,
[BINARY_OP_SUBSCR_GETITEM] = _RECORD_TOS_INDEX,
[RETURN_VALUE] = _RECORD_CALLER_CODE_INDEX,
[YIELD_VALUE] = _RECORD_CALLER_CODE_INDEX,
[LOAD_ATTR_INSTANCE_VALUE] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_WITH_HINT] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_SLOT] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_PROPERTY] = _RECORD_TOS_TYPE_INDEX,
[STORE_ATTR_WITH_HINT] = _RECORD_TOS_TYPE_INDEX,
[STORE_ATTR_SLOT] = _RECORD_TOS_TYPE_INDEX,
[FOR_ITER_GEN] = _RECORD_NOS_INDEX,
[LOAD_ATTR_METHOD_WITH_VALUES] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_METHOD_NO_DICT] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _RECORD_TOS_TYPE_INDEX,
[LOAD_ATTR_METHOD_LAZY_DICT] = _RECORD_TOS_TYPE_INDEX,
[CALL_PY_GENERAL] = _RECORD_CALLABLE_INDEX,
[CALL_BOUND_METHOD_GENERAL] = _RECORD_BOUND_METHOD_INDEX,
[CALL_NON_PY_GENERAL] = _RECORD_CALLABLE_INDEX,
[CALL_BOUND_METHOD_EXACT_ARGS] = _RECORD_BOUND_METHOD_INDEX,
[CALL_PY_EXACT_ARGS] = _RECORD_CALLABLE_INDEX,
[CALL_ALLOC_AND_ENTER_INIT] = _RECORD_CALLABLE_INDEX,
[CALL_BUILTIN_CLASS] = _RECORD_CALLABLE_INDEX,
[CALL_BUILTIN_O] = _RECORD_CALLABLE_INDEX,
[CALL_BUILTIN_FAST] = _RECORD_CALLABLE_INDEX,
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = _RECORD_CALLABLE_INDEX,
[CALL_METHOD_DESCRIPTOR_O] = _RECORD_CALLABLE_INDEX,
[CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _RECORD_CALLABLE_INDEX,
[CALL_METHOD_DESCRIPTOR_NOARGS] = _RECORD_CALLABLE_INDEX,
[CALL_EX_PY] = _RECORD_4OS_INDEX,
[RETURN_GENERATOR] = _RECORD_CALLER_CODE_INDEX,
};
const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[8] = {
[0] = NULL,
[_RECORD_TOS_TYPE_INDEX] = _PyOpcode_RecordFunction_TOS_TYPE,
[_RECORD_TOS_INDEX] = _PyOpcode_RecordFunction_TOS,
[_RECORD_CALLER_CODE_INDEX] = _PyOpcode_RecordFunction_CALLER_CODE,
[_RECORD_NOS_INDEX] = _PyOpcode_RecordFunction_NOS,
[_RECORD_CALLABLE_INDEX] = _PyOpcode_RecordFunction_CALLABLE,
[_RECORD_BOUND_METHOD_INDEX] = _PyOpcode_RecordFunction_BOUND_METHOD,
[_RECORD_4OS_INDEX] = _PyOpcode_RecordFunction_4OS,
};