Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-14 07:41:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-127958: Trace from RESUME in the JIT (GH-145905)

Browse source
This commit is contained in:
Ken Jin 2026-03-17 00:18:59 +08:00 • committed by GitHub
parent 81ef1b7317
commit 3d0824aef2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 915 additions and 597 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Python/assemble.c
View file

@ -669,7 +669,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_
// The offset (in code units) of the END_SEND from the SEND in the `yield from` sequence.
#define END_SEND_OFFSET 5
#define END_SEND_OFFSET 6
static int
resolve_jump_offsets(instr_sequence *instrs)

50
Python/bytecodes.c
View file

@ -148,8 +148,9 @@ dummy_func(
pure inst(NOP, (--)) {
}
family(RESUME, 0) = {
family(RESUME, 1) = {
RESUME_CHECK,
RESUME_CHECK_JIT,
};
macro(NOT_TAKEN) = NOP;
@ -171,12 +172,8 @@ dummy_func(
}
}
op(_QUICKEN_RESUME, (--)) {
#if ENABLE_SPECIALIZATION
if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
}
#endif /* ENABLE_SPECIALIZATION */
tier1 op(_QUICKEN_RESUME, (counter/1 --)) {
_Py_Specialize_Resume(this_instr, tstate, frame);
}
tier1 op(_MAYBE_INSTRUMENT, (--)) {
@ -226,7 +223,11 @@ dummy_func(
_QUICKEN_RESUME +
_CHECK_PERIODIC_IF_NOT_YIELD_FROM;
inst(RESUME_CHECK, (--)) {
macro(RESUME_CHECK) =
unused/1 +
_RESUME_CHECK;
op(_RESUME_CHECK, (--)) {
#if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
@ -241,6 +242,11 @@ dummy_func(
#endif
}
macro(RESUME_CHECK_JIT) =
unused/1 +
_RESUME_CHECK +
_JIT;
op(_MONITOR_RESUME, (--)) {
int err = _Py_call_instrumentation(
tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr);
@ -252,6 +258,7 @@ dummy_func(
}
macro(INSTRUMENTED_RESUME) =
unused/1 +
_LOAD_BYTECODE +
_MAYBE_INSTRUMENT +
_CHECK_PERIODIC_IF_NOT_YIELD_FROM +
@ -3114,9 +3121,11 @@ dummy_func(
tier1 op(_JIT, (--)) {
#ifdef _Py_TIER2
bool is_resume = this_instr->op.code == RESUME_CHECK_JIT;
_Py_BackoffCounter counter = this_instr[1].counter;
if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) &&
this_instr->op.code == JUMP_BACKWARD_JIT &&
if ((backoff_counter_triggers(counter) &&
!IS_JIT_TRACING() &&
(this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) &&
next_instr->op.code != ENTER_EXECUTOR) {
/* Back up over EXTENDED_ARGs so executor is inserted at the correct place */
_Py_CODEUNIT *insert_exec_at = this_instr;
@ -3124,7 +3133,8 @@ dummy_func(
oparg >>= 8;
insert_exec_at--;
}
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, stack_pointer, 0, NULL, oparg, NULL);
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at,
is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL);
if (succ) {
ENTER_TRACING();
}
@ -3175,12 +3185,22 @@ dummy_func(
tier1 inst(ENTER_EXECUTOR, (--)) {
#ifdef _Py_TIER2
if (IS_JIT_TRACING()) {
next_instr = this_instr;
goto stop_tracing;
}
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
if (IS_JIT_TRACING()) {
int og_opcode = executor->vm_data.opcode;
int og_oparg = (oparg & ~255) | executor->vm_data.oparg;
next_instr = this_instr;
if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) {
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
opcode = og_opcode;
oparg = og_oparg;
DISPATCH_GOTO_NON_TRACING();
}
goto stop_tracing;
}
assert(executor->vm_data.index == INSTR_OFFSET() - 1);
assert(executor->vm_data.code == code);
assert(executor->vm_data.valid);

5
Python/ceval.c
View file

@ -1095,7 +1095,8 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
{ .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */
{ .op.code = NOP, .op.arg = 0 },
{ .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */
{ .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START }
{ .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START },
{ .op.code = CACHE, .op.arg = 0 } /* RESUME's CACHE */
};
const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS;
@ -1371,7 +1372,7 @@ _PyTier2Interpreter(
for (;;) {
uopcode = next_uop->opcode;
#ifdef Py_DEBUG
if (frame->lltrace >= 3) {
if (frame->lltrace >= 4) {
dump_stack(frame, stack_pointer);
printf(" cache=[");
dump_cache_item(_tos_cache0, 0, current_cached_values);

2
Python/executor_cases.c.h generated
View file

@ -93,8 +93,6 @@
break;
}
/* _QUICKEN_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _RESUME_CHECK_r00: {

119
Python/generated_cases.c.h generated
View file

@ -5691,12 +5691,22 @@
INSTRUCTION_STATS(ENTER_EXECUTOR);
opcode = ENTER_EXECUTOR;
#ifdef _Py_TIER2
if (IS_JIT_TRACING()) {
next_instr = this_instr;
JUMP_TO_LABEL(stop_tracing);
}
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
if (IS_JIT_TRACING()) {
int og_opcode = executor->vm_data.opcode;
int og_oparg = (oparg & ~255) | executor->vm_data.oparg;
next_instr = this_instr;
if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) {
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
opcode = og_opcode;
oparg = og_oparg;
DISPATCH_GOTO_NON_TRACING();
}
JUMP_TO_LABEL(stop_tracing);
}
assert(executor->vm_data.index == INSTR_OFFSET() - 1);
assert(executor->vm_data.code == code);
assert(executor->vm_data.valid);
@ -7436,8 +7446,9 @@
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
/* Skip 1 cache entry */
// _LOAD_BYTECODE
{
#ifdef Py_GIL_DISABLED
@ -7785,16 +7796,19 @@
// _JIT
{
#ifdef _Py_TIER2
bool is_resume = this_instr->op.code == RESUME_CHECK_JIT;
_Py_BackoffCounter counter = this_instr[1].counter;
if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) &&
this_instr->op.code == JUMP_BACKWARD_JIT &&
if ((backoff_counter_triggers(counter) &&
!IS_JIT_TRACING() &&
(this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) &&
next_instr->op.code != ENTER_EXECUTOR) {
_Py_CODEUNIT *insert_exec_at = this_instr;
while (oparg > 255) {
oparg >>= 8;
insert_exec_at--;
}
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, stack_pointer, 0, NULL, oparg, NULL);
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at,
is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL);
if (succ) {
ENTER_TRACING();
}
@ -10446,10 +10460,10 @@
(void)(opcode);
#endif
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(RESUME);
PREDICTED_RESUME:;
_Py_CODEUNIT* const this_instr = next_instr - 1;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
// _LOAD_BYTECODE
{
@ -10496,11 +10510,11 @@
}
// _QUICKEN_RESUME
{
#if ENABLE_SPECIALIZATION
if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
}
#endif /* ENABLE_SPECIALIZATION */
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_Resume(this_instr, tstate, frame);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
// _CHECK_PERIODIC_IF_NOT_YIELD_FROM
{
@ -10524,9 +10538,10 @@
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(RESUME_CHECK);
static_assert(0 == 0, "incorrect cache size");
static_assert(1 == 1, "incorrect cache size");
/* Skip 1 cache entry */
#if defined(__EMSCRIPTEN__)
if (_Py_emscripten_signal_clock == 0) {
UPDATE_MISS_STATS(RESUME);
@ -10554,6 +10569,76 @@
DISPATCH();
}
TARGET(RESUME_CHECK_JIT) {
#if _Py_TAIL_CALL_INTERP
int opcode = RESUME_CHECK_JIT;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(RESUME_CHECK_JIT);
static_assert(1 == 1, "incorrect cache size");
/* Skip 1 cache entry */
// _RESUME_CHECK
{
#if defined(__EMSCRIPTEN__)
if (_Py_emscripten_signal_clock == 0) {
UPDATE_MISS_STATS(RESUME);
assert(_PyOpcode_Deopt[opcode] == (RESUME));
JUMP_TO_PREDICTED(RESUME);
}
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
if (eval_breaker != version) {
UPDATE_MISS_STATS(RESUME);
assert(_PyOpcode_Deopt[opcode] == (RESUME));
JUMP_TO_PREDICTED(RESUME);
}
#ifdef Py_GIL_DISABLED
if (frame->tlbc_index !=
((_PyThreadStateImpl *)tstate)->tlbc_index) {
UPDATE_MISS_STATS(RESUME);
assert(_PyOpcode_Deopt[opcode] == (RESUME));
JUMP_TO_PREDICTED(RESUME);
}
#endif
}
// _JIT
{
#ifdef _Py_TIER2
bool is_resume = this_instr->op.code == RESUME_CHECK_JIT;
_Py_BackoffCounter counter = this_instr[1].counter;
if ((backoff_counter_triggers(counter) &&
!IS_JIT_TRACING() &&
(this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) &&
next_instr->op.code != ENTER_EXECUTOR) {
_Py_CODEUNIT *insert_exec_at = this_instr;
while (oparg > 255) {
oparg >>= 8;
insert_exec_at--;
}
int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at,
is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL);
if (succ) {
ENTER_TRACING();
}
else {
this_instr[1].counter = restart_backoff_counter(counter);
}
}
else {
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
#endif
}
DISPATCH();
}
TARGET(RETURN_GENERATOR) {
#if _Py_TAIL_CALL_INTERP
int opcode = RETURN_GENERATOR;

1
Python/instrumentation.c
View file

@ -576,6 +576,7 @@ sanity_check_instrumentation(PyCodeObject *code)
CHECK(opcode != END_FOR);
CHECK(opcode != RESUME);
CHECK(opcode != RESUME_CHECK);
CHECK(opcode != RESUME_CHECK_JIT);
CHECK(opcode != INSTRUMENTED_RESUME);
if (!is_instrumented(opcode)) {
CHECK(_PyOpcode_Deopt[opcode] == opcode);

9
Python/opcode_targets.h generated
View file

@ -198,6 +198,7 @@ static void *opcode_targets_table[256] = {
&&TARGET_LOAD_SUPER_ATTR_ATTR,
&&TARGET_LOAD_SUPER_ATTR_METHOD,
&&TARGET_RESUME_CHECK,
&&TARGET_RESUME_CHECK_JIT,
&&TARGET_SEND_GEN,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_STORE_ATTR_SLOT,
@ -232,7 +233,6 @@ static void *opcode_targets_table[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_POP_ITER,
&&TARGET_INSTRUMENTED_END_SEND,
@ -472,7 +472,7 @@ static void *opcode_tracing_targets_table[256] = {
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
@ -718,6 +718,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS);
@ -959,6 +960,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[RESERVED] = _TAIL_CALL_RESERVED,
[RESUME] = _TAIL_CALL_RESUME,
[RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK,
[RESUME_CHECK_JIT] = _TAIL_CALL_RESUME_CHECK_JIT,
[RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR,
[RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE,
[SEND] = _TAIL_CALL_SEND,
@ -1007,7 +1009,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[213] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,
[216] = _TAIL_CALL_UNKNOWN_OPCODE,
@ -1217,6 +1218,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[RESERVED] = _TAIL_CALL_TRACE_RECORD,
[RESUME] = _TAIL_CALL_TRACE_RECORD,
[RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD,
[RESUME_CHECK_JIT] = _TAIL_CALL_TRACE_RECORD,
[RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD,
[RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD,
[SEND] = _TAIL_CALL_TRACE_RECORD,
@ -1265,7 +1267,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[213] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,
[216] = _TAIL_CALL_UNKNOWN_OPCODE,

34
Python/optimizer.c
View file

@ -632,6 +632,12 @@ _PyJit_translate_single_bytecode_to_trace(
target--;
}
if (opcode == ENTER_EXECUTOR) {
_PyExecutorObject *executor = old_code->co_executors->executors[oparg & 255];
opcode = executor->vm_data.opcode;
oparg = (oparg & ~255) | executor->vm_data.oparg;
}
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] > 0) {
uint16_t backoff = (this_instr + 1)->counter.value_and_backoff;
// adaptive_counter_cooldown is a fresh specialization.
@ -823,6 +829,7 @@ _PyJit_translate_single_bytecode_to_trace(
case RESUME:
case RESUME_CHECK:
case RESUME_CHECK_JIT:
/* Use a special tier 2 version of RESUME_CHECK to allow traces to
* start with RESUME_CHECK */
ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target);
@ -1038,12 +1045,9 @@ _PyJit_TryInitializeTracing(
_Py_RecordFuncPtr record_func = _PyOpcode_RecordFunctions[record_func_index];
record_func(frame, stack_pointer, oparg, &tracer->prev_state.recorded_value);
}
assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL));
assert(curr_instr->op.code == JUMP_BACKWARD_JIT || curr_instr->op.code == RESUME_CHECK_JIT || (exit != NULL));
tracer->initial_state.jump_backward_instr = curr_instr;
if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) {
close_loop_instr[1].counter = trigger_backoff_counter();
}
tracer->is_tracing = true;
return 1;
}
@ -1063,7 +1067,12 @@ _PyJit_FinalizeTracing(PyThreadState *tstate, int err)
tracer->initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter);
}
else {
tracer->initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(&tstate->interp->opt_config);
if (tracer->initial_state.jump_backward_instr[0].op.code == JUMP_BACKWARD_JIT) {
tracer->initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(&tstate->interp->opt_config);
}
else {
tracer->initial_state.jump_backward_instr[1].counter = initial_resume_backoff_counter(&tstate->interp->opt_config);
}
}
}
else if (tracer->initial_state.executor->vm_data.valid) {
@ -1092,6 +1101,19 @@ _PyJit_FinalizeTracing(PyThreadState *tstate, int err)
tracer->is_tracing = false;
}
bool
_PyJit_EnterExecutorShouldStopTracing(int og_opcode)
{
// Continue tracing (skip over the executor). If it's a RESUME
// trace to form longer, more optimizeable traces.
// We want to trace over RESUME traces. Otherwise, functions with lots of RESUME
// end up with many fragmented traces which perform badly.
// See for example, the richards benchmark in pyperformance.
// For consideration: We may want to consider tracing over side traces
// inserted into bytecode as well in the future.
return og_opcode == RESUME_CHECK_JIT;
}
void
_PyJit_TracerFree(_PyThreadStateImpl *_tstate)
{
@ -1780,7 +1802,7 @@ _Py_ExecutorDetach(_PyExecutorObject *executor)
assert(instruction->op.code == ENTER_EXECUTOR);
int index = instruction->op.arg;
assert(code->co_executors->executors[index] == executor);
instruction->op.code = executor->vm_data.opcode;
instruction->op.code = _PyOpcode_Deopt[executor->vm_data.opcode];
instruction->op.arg = executor->vm_data.oparg;
executor->vm_data.code = NULL;
code->co_executors->executors[index] = NULL;

2
Python/optimizer_cases.c.h generated
View file

@ -17,8 +17,6 @@
break;
}
/* _QUICKEN_RESUME is not a viable micro-op for tier 2 */
/* _LOAD_BYTECODE is not a viable micro-op for tier 2 */
case _RESUME_CHECK: {

8
Python/pystate.c
View file

@ -607,11 +607,13 @@ init_interpreter(PyInterpreterState *interp,
// Initialize optimization configuration from environment variables
// PYTHON_JIT_STRESS sets aggressive defaults for testing, but can be overridden
uint16_t jump_default = JUMP_BACKWARD_INITIAL_VALUE;
uint16_t resume_default = RESUME_INITIAL_VALUE;
uint16_t side_exit_default = SIDE_EXIT_INITIAL_VALUE;
if (is_env_enabled("PYTHON_JIT_STRESS")) {
jump_default = 63;
side_exit_default = 63;
resume_default = 127;
}
init_policy(&interp->opt_config.jump_backward_initial_value,
@ -620,6 +622,12 @@ init_interpreter(PyInterpreterState *interp,
init_policy(&interp->opt_config.jump_backward_initial_backoff,
"PYTHON_JIT_JUMP_BACKWARD_INITIAL_BACKOFF",
JUMP_BACKWARD_INITIAL_BACKOFF, 0, MAX_BACKOFF);
init_policy(&interp->opt_config.resume_initial_value,
"PYTHON_JIT_RESUME_INITIAL_VALUE",
resume_default, 1, MAX_VALUE);
init_policy(&interp->opt_config.resume_initial_backoff,
"PYTHON_JIT_RESUME_INITIAL_BACKOFF",
RESUME_INITIAL_BACKOFF, 0, MAX_BACKOFF);
init_policy(&interp->opt_config.side_exit_initial_value,
"PYTHON_JIT_SIDE_EXIT_INITIAL_VALUE",
side_exit_default, 1, MAX_VALUE);

30
Python/specialize.c
View file

@ -46,16 +46,18 @@ void
_PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters)
{
#if ENABLE_SPECIALIZATION
_Py_BackoffCounter jump_counter, adaptive_counter;
_Py_BackoffCounter jump_counter, adaptive_counter, resume_counter;
if (enable_counters) {
PyThreadState *tstate = _PyThreadState_GET();
PyInterpreterState *interp = tstate->interp;
jump_counter = initial_jump_backoff_counter(&interp->opt_config);
adaptive_counter = adaptive_counter_warmup();
resume_counter = initial_resume_backoff_counter(&interp->opt_config);
}
else {
jump_counter = initial_unreachable_backoff_counter();
adaptive_counter = initial_unreachable_backoff_counter();
resume_counter = initial_unreachable_backoff_counter();
}
int opcode = 0;
int oparg = 0;
@ -70,6 +72,9 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters
case JUMP_BACKWARD:
instructions[i + 1].counter = jump_counter;
break;
case RESUME:
instructions[i + 1].counter = resume_counter;
break;
case POP_JUMP_IF_FALSE:
case POP_JUMP_IF_TRUE:
case POP_JUMP_IF_NONE:
@ -2781,6 +2786,28 @@ _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr)
return;
}
void
_Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame)
{
if (tstate->tracing == 0 && instr->op.code == RESUME) {
if (tstate->interp->jit) {
PyCodeObject *co = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);
if (co != NULL &&
PyCode_Check(co) &&
(co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == 0) {
specialize(instr, RESUME_CHECK_JIT);
set_counter((_Py_BackoffCounter *)instr + 1, initial_resume_backoff_counter(&tstate->interp->opt_config));
return;
}
}
specialize(instr, RESUME_CHECK);
return;
}
unspecialize(instr);
return;
}
#ifdef Py_STATS
void
_Py_GatherStats_GetIter(_PyStackRef iterable)
@ -2883,5 +2910,6 @@ const struct _PyCode8 _Py_InitCleanup = {
EXIT_INIT_CHECK, 0,
RETURN_VALUE, 0,
RESUME, RESUME_AT_FUNC_START,
CACHE, 0, /* RESUME's cache */
}
};