Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-01 06:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-132554: Fix tier2 FOR_ITER implementation and optimizations (GH-135137)

Browse source
This commit is contained in:
Mark Shannon 2025-06-05 18:53:57 +01:00 • committed by GitHub
parent d9cad074d5
commit b90ecea9e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 155 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

97
Python/bytecodes.c
View file

@ -3125,100 +3125,49 @@ dummy_func(
}
replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
ERROR_NO_POP();
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
}
else {
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (!matches) {
ERROR_NO_POP();
}
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
/* Jump forward oparg, then skip following END_FOR */
JUMPBY(oparg + 1);
DISPATCH();
}
next = PyStackRef_FromPyObjectSteal(next_o);
// Jump forward by oparg and skip the following END_FOR
JUMPBY(oparg + 1);
DISPATCH();
}
next = item;
}
op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(!PyStackRef_IsNull(null_or_index));
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (!matches) {
ERROR_NO_POP();
}
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
_PyErr_Clear(tstate);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
ERROR_NO_POP();
}
/* iterator ended normally */
/* The translator sets the deopt target just past the matching END_FOR */
EXIT_IF(true);
}
next = PyStackRef_FromPyObjectSteal(next_o);
// Common case: no jump, leave it to the code generator
next = item;
}
macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o != NULL) {
next = PyStackRef_FromPyObjectSteal(next_o);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (!matches) {
ERROR_NO_POP();
}
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
/* Skip END_FOR */
JUMPBY(oparg + 1);
DISPATCH();
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
ERROR_NO_POP();
}
// Jump forward by oparg and skip the following END_FOR
JUMPBY(oparg + 1);
DISPATCH();
}
next = item;
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(iter_o) != &PyList_Type);

28
Python/ceval.c
View file

@ -3439,8 +3439,8 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na
return value;
}
_PyStackRef
_PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index)
static _PyStackRef
foriter_next(PyObject *seq, _PyStackRef index)
{
assert(PyStackRef_IsTaggedInt(index));
assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq));
@ -3459,6 +3459,30 @@ _PyForIter_NextWithIndex(PyObject *seq, _PyStackRef index)
return PyStackRef_FromPyObjectSteal(item);
}
_PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef* index_ptr)
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyStackRef index = *index_ptr;
if (PyStackRef_IsTaggedInt(index)) {
*index_ptr = PyStackRef_IncrementTaggedIntNoOverflow(index);
return foriter_next(iter_o, index);
}
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
_PyErr_Clear(tstate);
}
else {
return PyStackRef_ERROR;
}
}
return PyStackRef_NULL;
}
return PyStackRef_FromPyObjectSteal(next_o);
}
/* Check if a 'cls' provides the given special method. */
static inline int
type_has_special_method(PyTypeObject *cls, PyObject *name)

25
Python/executor_cases.c.h generated
View file

@ -4268,33 +4268,20 @@
_PyStackRef next;
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyStackRef_IsNull(null_or_index)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
JUMP_TO_ERROR();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
_PyErr_Clear(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
next = item;
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());

91
Python/generated_cases.c.h generated
View file

@ -5753,41 +5753,18 @@
}
// _FOR_ITER
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
JUMP_TO_LABEL(error);
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
JUMPBY(oparg + 1);
DISPATCH();
}
next = PyStackRef_FromPyObjectSteal(next_o);
JUMPBY(oparg + 1);
stack_pointer[-1] = null_or_index;
DISPATCH();
}
next = item;
}
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
@ -7059,45 +7036,19 @@
/* Skip 1 cache entry */
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (PyStackRef_IsTaggedInt(null_or_index)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
next = _PyForIter_NextWithIndex(iter_o, null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(next)) {
JUMPBY(oparg + 1);
DISPATCH();
}
null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o != NULL) {
next = PyStackRef_FromPyObjectSteal(next_o);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
}
else {
if (_PyErr_Occurred(tstate)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!matches) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
JUMPBY(oparg + 1);
DISPATCH();
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
JUMP_TO_LABEL(error);
}
JUMPBY(oparg + 1);
stack_pointer[-1] = null_or_index;
DISPATCH();
}
next = item;
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;

11
Python/optimizer_bytecodes.c
View file

@ -840,6 +840,17 @@ dummy_func(void) {
value = sym_new_unknown(ctx);
}
op(_GET_ITER, (iterable -- iter, index_or_null)) {
if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
iter = iterable;
index_or_null = sym_new_not_null(ctx);
}
else {
iter = sym_new_not_null(ctx);
index_or_null = sym_new_unknown(ctx);
}
}
op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame: _Py_UOpsAbstractFrame*)) {
gen_frame = NULL;
/* We are about to hit the end of the trace */

12
Python/optimizer_cases.c.h generated
View file

@ -1562,10 +1562,18 @@
}
case _GET_ITER: {
JitOptSymbol *iterable;
JitOptSymbol *iter;
JitOptSymbol *index_or_null;
iter = sym_new_not_null(ctx);
index_or_null = sym_new_not_null(ctx);
iterable = stack_pointer[-1];
if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
iter = iterable;
index_or_null = sym_new_not_null(ctx);
}
else {
iter = sym_new_not_null(ctx);
index_or_null = sym_new_unknown(ctx);
}
stack_pointer[-1] = iter;
stack_pointer[0] = index_or_null;
stack_pointer += 1;

4
Python/stackrefs.c
View file

@ -40,6 +40,7 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber)
PyObject *
_Py_stackref_get_object(_PyStackRef ref)
{
assert(!PyStackRef_IsError(ref));
if (ref.index == 0) {
return NULL;
}
@ -64,6 +65,7 @@ PyStackRef_Is(_PyStackRef a, _PyStackRef b)
PyObject *
_Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
PyInterpreterState *interp = PyInterpreterState_Get();
if (ref.index >= interp->next_stackref) {
_Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber);
@ -128,6 +130,7 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber)
void
_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber)
{
assert(!PyStackRef_IsError(ref));
if (ref.index < INITIAL_STACKREF_INDEX) {
return;
}
@ -152,6 +155,7 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber
void
_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref)
{
assert(!PyStackRef_IsError(ref));
assert(ref.index < INITIAL_STACKREF_INDEX);
TableEntry *entry = make_table_entry(obj, "builtin-object", 0);
if (entry == NULL) {