Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-20 10:51:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-145668: Add FOR_ITER specialization for virtual iterators. Specialize GET_ITER. (GH-147967)

Browse source 
* Add FOR_ITER_VIRTUAL to specialize FOR_ITER for virtual iterators
* Add GET_ITER_SELF to specialize GET_ITER for iterators (including generators)
* Add GET_ITER_VIRTUAL to specialize GET_ITER for iterables as virtual iterators
* Add new (internal) _tp_iteritem function slot to PyTypeObject
* Put limited RESUME at start of genexpr for free-threading. Fix up exception handling in genexpr
This commit is contained in:
Mark Shannon 2026-04-16 15:22:22 +01:00 • committed by GitHub
parent 0fcf2b72d3
commit 600f4dbd54
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 2987 additions and 1534 deletions
Download patch file Download diff file Expand all files Collapse all files

117
Python/bytecodes.c
View file

@ -199,7 +199,7 @@ dummy_func(
}
}
op(_LOAD_BYTECODE, (--)) {
replaced op(_LOAD_BYTECODE, (--)) {
#ifdef Py_GIL_DISABLED
if (frame->tlbc_index !=
((_PyThreadStateImpl *)tstate)->tlbc_index) {
@ -2820,6 +2820,11 @@ dummy_func(
}
}
op(_GUARD_TYPE, (type/4, owner -- owner)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
EXIT_IF(tp != (PyTypeObject *)type);
}
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_dictoffset < 0);
@ -3655,16 +3660,71 @@ dummy_func(
values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o);
}
inst(GET_ITER, (iterable -- iter, index_or_null)) {
#ifdef Py_STATS
_Py_GatherStats_GetIter(iterable);
#endif
family(GET_ITER, INLINE_CACHE_ENTRIES_GET_ITER) = {
GET_ITER_SELF,
GET_ITER_VIRTUAL,
};
specializing op(_SPECIALIZE_GET_ITER, (counter/1, iterable -- iterable)) {
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_GetIter(iterable, next_instr);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(GET_ITER);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
op(_GET_ITER, (iterable -- iter, index_or_null)) {
_PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg);
DEAD(iterable);
ERROR_IF(PyStackRef_IsError(result));
iter = result;
}
macro(GET_ITER) =
_RECORD_TOS_TYPE +
_SPECIALIZE_GET_ITER +
_GET_ITER;
op(_GUARD_ITERATOR, (iterable -- iterable)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
EXIT_IF(tp->tp_iter != PyObject_SelfIter);
STAT_INC(GET_ITER, hit);
}
macro(GET_ITER_SELF) =
_RECORD_TOS_TYPE +
unused/1 +
_GUARD_ITERATOR +
PUSH_NULL;
op(_GUARD_ITER_VIRTUAL, (iterable -- iterable)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
EXIT_IF(tp->_tp_iteritem == NULL);
STAT_INC(GET_ITER, hit);
}
op(_PUSH_TAGGED_ZERO, ( -- zero)) {
zero = PyStackRef_TagInt(0);
}
macro(GET_ITER_VIRTUAL) =
_RECORD_TOS_TYPE +
unused/1 +
_GUARD_ITER_VIRTUAL +
_PUSH_TAGGED_ZERO;
op(_GET_ITER_TRAD, (iterable -- iter, index_or_null)) {
PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
PyStackRef_CLOSE(iterable);
ERROR_IF(iter_o == NULL);
iter = PyStackRef_FromPyObjectSteal(iter_o);
index_or_null = PyStackRef_NULL;
}
// Most members of this family are "secretly" super-instructions.
// When the loop is exhausted, they jump, and the jump target is
// always END_FOR, which pops two values off the stack.
@ -3676,6 +3736,7 @@ dummy_func(
FOR_ITER_TUPLE,
FOR_ITER_RANGE,
FOR_ITER_GEN,
FOR_ITER_VIRTUAL,
};
specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter, null_or_index -- iter, null_or_index)) {
@ -3703,6 +3764,8 @@ dummy_func(
next = item;
}
macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) {
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);
if (!PyStackRef_IsValid(item)) {
@ -3716,9 +3779,51 @@ dummy_func(
next = item;
}
op(_GUARD_NOS_ITER_VIRTUAL, (iter, null_or_index -- iter, null_or_index)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(iter_o)->_tp_iteritem == NULL);
}
macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
replaced op(_FOR_ITER_VIRTUAL, (iter, null_or_index -- iter, null_or_index, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
ERROR_NO_POP();
}
// Jump forward by oparg and skip the following END_FOR
JUMPBY(oparg + 1);
DISPATCH();
}
null_or_index = PyStackRef_TagInt(index);
next = PyStackRef_FromPyObjectSteal(next_o);
}
macro(FOR_ITER_VIRTUAL) =
unused/1 + // Skip over the counter
_GUARD_NOS_ITER_VIRTUAL +
_FOR_ITER_VIRTUAL;
op(_FOR_ITER_VIRTUAL_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
ERROR_NO_POP();
}
/* iterator ended normally */
/* The translator sets the deopt target just past the matching END_FOR */
EXIT_IF(true);
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
}
inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) {
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index);

40
Python/ceval.c
View file

@ -1115,7 +1115,7 @@ _PyStackRef
_PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from)
{
PyTypeObject *tp = PyStackRef_TYPE(iterable);
if (tp == &PyTuple_Type || tp == &PyList_Type) {
if (tp->_tp_iteritem != NULL) {
/* Leave iterable on stack and pushed tagged 0 */
*index_or_null = PyStackRef_TagInt(0);
return iterable;
@ -3697,36 +3697,24 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na
return value;
}
static _PyStackRef
foriter_next(PyObject *seq, _PyStackRef index)
{
assert(PyStackRef_IsTaggedInt(index));
assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq));
intptr_t i = PyStackRef_UntagInt(index);
if (PyTuple_CheckExact(seq)) {
size_t size = PyTuple_GET_SIZE(seq);
if ((size_t)i >= size) {
return PyStackRef_NULL;
}
return PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, i));
}
PyObject *item = _PyList_GetItemRef((PyListObject *)seq, i);
if (item == NULL) {
return PyStackRef_NULL;
}
return PyStackRef_FromPyObjectSteal(item);
}
_PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef* index_ptr)
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyStackRef index = *index_ptr;
if (PyStackRef_IsTaggedInt(index)) {
*index_ptr = PyStackRef_IncrementTaggedIntNoOverflow(index);
return foriter_next(iter_o, index);
intptr_t i = PyStackRef_UntagInt(index);
assert(i >= 0);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, i);
i = next_index.index;
PyObject *next = next_index.object;
if (next == NULL) {
return i < 0 ? PyStackRef_ERROR : PyStackRef_NULL;
}
*index_ptr = PyStackRef_TagInt(i);
return PyStackRef_FromPyObjectSteal(next);
}
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next_o == NULL) {
PyObject *next = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
_PyEval_MonitorRaise(tstate, frame, frame->instr_ptr);
@ -3738,7 +3726,7 @@ _PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreter
}
return PyStackRef_NULL;
}
return PyStackRef_FromPyObjectSteal(next_o);
return PyStackRef_FromPyObjectSteal(next);
}
/* Check if a 'cls' provides the given special method. */

15
Python/codegen.c
View file

@ -1224,12 +1224,17 @@ codegen_wrap_in_stopiteration_handler(compiler *c)
{
NEW_JUMP_TARGET_LABEL(c, handler);
/* Insert SETUP_CLEANUP just before RESUME */
/* Insert SETUP_CLEANUP just after the initial RETURN_GENERATOR; POP_TOP */
instr_sequence *seq = INSTR_SEQUENCE(c);
int resume = 0;
while (_PyInstructionSequence_GetInstruction(seq, resume).i_opcode != RESUME) {
while (_PyInstructionSequence_GetInstruction(seq, resume).i_opcode != RETURN_GENERATOR) {
resume++;
assert(resume < seq->s_used);
}
resume++;
assert(_PyInstructionSequence_GetInstruction(seq, resume).i_opcode == POP_TOP);
resume++;
assert(resume < seq->s_used);
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
seq, resume,
@ -4977,10 +4982,14 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
INSTR_SEQUENCE(c), 0,
LOAD_FAST, 0, LOC(outermost->iter)));
RESUME, RESUME_AT_GEN_EXPR_START, NO_LOCATION));
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
INSTR_SEQUENCE(c), 1,
LOAD_FAST, 0, LOC(outermost->iter)));
RETURN_IF_ERROR(
_PyInstructionSequence_InsertInstruction(
INSTR_SEQUENCE(c), 2,
outermost->is_async ? GET_AITER : GET_ITER,
0, LOC(outermost->iter)));
iter_state = ITERATOR_ON_STACK;

483
Python/executor_cases.c.h generated
View file

@ -93,7 +93,7 @@
break;
}
/* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it is replaced */
case _RESUME_CHECK_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
@ -11271,6 +11271,95 @@
break;
}
case _GUARD_TYPE_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef owner;
owner = stack_pointer[-1];
PyObject *type = (PyObject *)CURRENT_OPERAND0_64();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
if (tp != (PyTypeObject *)type) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = owner;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TYPE_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
PyObject *type = (PyObject *)CURRENT_OPERAND0_64();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
if (tp != (PyTypeObject *)type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = owner;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = owner;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TYPE_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
PyObject *type = (PyObject *)CURRENT_OPERAND0_64();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
if (tp != (PyTypeObject *)type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = owner;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = owner;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TYPE_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
PyObject *type = (PyObject *)CURRENT_OPERAND0_64();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
if (tp != (PyTypeObject *)type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = owner;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = owner;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
@ -13620,9 +13709,6 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
iterable = _stack_item_0;
#ifdef Py_STATS
_Py_GatherStats_GetIter(iterable);
#endif
stack_pointer[0] = iterable;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@ -13646,6 +13732,256 @@
break;
}
case _GUARD_ITERATOR_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
iterable = stack_pointer[-1];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->tp_iter != PyObject_SelfIter) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache0 = iterable;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITERATOR_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef _stack_item_0 = _tos_cache0;
iterable = _stack_item_0;
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->tp_iter != PyObject_SelfIter) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = iterable;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache0 = iterable;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITERATOR_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
iterable = _stack_item_1;
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->tp_iter != PyObject_SelfIter) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = iterable;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache1 = iterable;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITERATOR_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
iterable = _stack_item_2;
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->tp_iter != PyObject_SelfIter) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = iterable;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache2 = iterable;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITER_VIRTUAL_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
iterable = stack_pointer[-1];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache0 = iterable;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITER_VIRTUAL_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef _stack_item_0 = _tos_cache0;
iterable = _stack_item_0;
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = iterable;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache0 = iterable;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITER_VIRTUAL_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
iterable = _stack_item_1;
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = iterable;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache1 = iterable;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_ITER_VIRTUAL_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
iterable = _stack_item_2;
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = iterable;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(GET_ITER, hit);
_tos_cache2 = iterable;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _PUSH_TAGGED_ZERO_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef zero;
zero = PyStackRef_TagInt(0);
_tos_cache0 = zero;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _PUSH_TAGGED_ZERO_r12: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef zero;
_PyStackRef _stack_item_0 = _tos_cache0;
zero = PyStackRef_TagInt(0);
_tos_cache1 = zero;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _PUSH_TAGGED_ZERO_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef zero;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
zero = PyStackRef_TagInt(0);
_tos_cache2 = zero;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GET_ITER_TRAD_r12: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iterable;
_PyStackRef iter;
_PyStackRef index_or_null;
_PyStackRef _stack_item_0 = _tos_cache0;
iterable = _stack_item_0;
stack_pointer[0] = iterable;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable));
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (iter_o == NULL) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
iter = PyStackRef_FromPyObjectSteal(iter_o);
index_or_null = PyStackRef_NULL;
_tos_cache1 = index_or_null;
_tos_cache0 = iter;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO_r23: {
@ -13692,6 +14028,145 @@
break;
}
case _GUARD_NOS_ITER_VIRTUAL_r02: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o)->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_ITER_VIRTUAL_r12: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
_PyStackRef _stack_item_0 = _tos_cache0;
iter = stack_pointer[-1];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o)->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = _stack_item_0;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_ITER_VIRTUAL_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
iter = _stack_item_0;
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o)->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = _stack_item_1;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_ITER_VIRTUAL_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
iter = _stack_item_1;
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o)->_tp_iteritem == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = iter;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = iter;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
/* _FOR_ITER_VIRTUAL is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_VIRTUAL_TIER_TWO_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
null_or_index = _stack_item_1;
iter = _stack_item_0;
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
stack_pointer[0] = iter;
stack_pointer[1] = null_or_index;
stack_pointer += 2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
_tos_cache2 = next;
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST_r02: {

2
Python/flowgraph.c
View file

@ -963,7 +963,7 @@ label_exception_targets(basicblock *entryblock) {
}
else if (instr->i_opcode == RESUME) {
instr->i_except = handler;
if (instr->i_oparg != RESUME_AT_FUNC_START) {
if (instr->i_oparg != RESUME_AT_FUNC_START && instr->i_oparg != RESUME_AT_GEN_EXPR_START) {
assert(last_yield_except_depth >= 0);
if (last_yield_except_depth == 1) {
instr->i_oparg |= RESUME_OPARG_DEPTH1_MASK;

163
Python/generated_cases.c.h generated
View file

@ -6379,6 +6379,58 @@
DISPATCH();
}
TARGET(FOR_ITER_VIRTUAL) {
#if _Py_TAIL_CALL_INTERP
int opcode = FOR_ITER_VIRTUAL;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_VIRTUAL);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
_PyStackRef iter;
_PyStackRef null_or_index;
_PyStackRef next;
/* Skip 1 cache entry */
// _GUARD_NOS_ITER_VIRTUAL
{
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o)->_tp_iteritem == NULL) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
}
// _FOR_ITER_VIRTUAL
{
null_or_index = stack_pointer[-1];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
JUMP_TO_LABEL(error);
}
JUMPBY(oparg + 1);
DISPATCH();
}
null_or_index = PyStackRef_TagInt(index);
next = PyStackRef_FromPyObjectSteal(next_o);
}
stack_pointer[-1] = null_or_index;
stack_pointer[0] = next;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
TARGET(GET_AITER) {
#if _Py_TAIL_CALL_INTERP
int opcode = GET_AITER;
@ -6499,22 +6551,41 @@
(void)(opcode);
#endif
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(GET_ITER);
PREDICTED_GET_ITER:;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
_PyStackRef iterable;
_PyStackRef iter;
_PyStackRef index_or_null;
iterable = stack_pointer[-1];
#ifdef Py_STATS
_Py_GatherStats_GetIter(iterable);
#endif
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsError(result)) {
JUMP_TO_LABEL(pop_1_error);
// _SPECIALIZE_GET_ITER
{
iterable = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_GetIter(iterable, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(GET_ITER);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _GET_ITER
{
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsError(result)) {
JUMP_TO_LABEL(pop_1_error);
}
iter = result;
}
iter = result;
stack_pointer[-1] = iter;
stack_pointer[0] = index_or_null;
stack_pointer += 1;
@ -6522,6 +6593,76 @@
DISPATCH();
}
TARGET(GET_ITER_SELF) {
#if _Py_TAIL_CALL_INTERP
int opcode = GET_ITER_SELF;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(GET_ITER_SELF);
static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size");
_PyStackRef iterable;
_PyStackRef res;
/* Skip 1 cache entry */
// _GUARD_ITERATOR
{
iterable = stack_pointer[-1];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->tp_iter != PyObject_SelfIter) {
UPDATE_MISS_STATS(GET_ITER);
assert(_PyOpcode_Deopt[opcode] == (GET_ITER));
JUMP_TO_PREDICTED(GET_ITER);
}
STAT_INC(GET_ITER, hit);
}
// _PUSH_NULL
{
res = PyStackRef_NULL;
}
stack_pointer[0] = res;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
TARGET(GET_ITER_VIRTUAL) {
#if _Py_TAIL_CALL_INTERP
int opcode = GET_ITER_VIRTUAL;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(GET_ITER_VIRTUAL);
static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size");
_PyStackRef iterable;
_PyStackRef zero;
/* Skip 1 cache entry */
// _GUARD_ITER_VIRTUAL
{
iterable = stack_pointer[-1];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable));
if (tp->_tp_iteritem == NULL) {
UPDATE_MISS_STATS(GET_ITER);
assert(_PyOpcode_Deopt[opcode] == (GET_ITER));
JUMP_TO_PREDICTED(GET_ITER);
}
STAT_INC(GET_ITER, hit);
}
// _PUSH_TAGGED_ZERO
{
zero = PyStackRef_TagInt(0);
}
stack_pointer[0] = zero;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
TARGET(GET_LEN) {
#if _Py_TAIL_CALL_INTERP
int opcode = GET_LEN;

27
Python/opcode_targets.h generated
View file

@ -178,6 +178,9 @@ static void *opcode_targets_table[256] = {
&&TARGET_FOR_ITER_LIST,
&&TARGET_FOR_ITER_RANGE,
&&TARGET_FOR_ITER_TUPLE,
&&TARGET_FOR_ITER_VIRTUAL,
&&TARGET_GET_ITER_SELF,
&&TARGET_GET_ITER_VIRTUAL,
&&TARGET_JUMP_BACKWARD_JIT,
&&TARGET_JUMP_BACKWARD_NO_JIT,
&&TARGET_LOAD_ATTR_CLASS,
@ -230,9 +233,6 @@ static void *opcode_targets_table[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_POP_ITER,
&&TARGET_INSTRUMENTED_END_SEND,
@ -473,9 +473,9 @@ static void *opcode_tracing_targets_table[256] = {
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
@ -621,10 +621,13 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS);
@ -862,10 +865,13 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST,
[FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE,
[FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE,
[FOR_ITER_VIRTUAL] = _TAIL_CALL_FOR_ITER_VIRTUAL,
[GET_AITER] = _TAIL_CALL_GET_AITER,
[GET_ANEXT] = _TAIL_CALL_GET_ANEXT,
[GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE,
[GET_ITER] = _TAIL_CALL_GET_ITER,
[GET_ITER_SELF] = _TAIL_CALL_GET_ITER_SELF,
[GET_ITER_VIRTUAL] = _TAIL_CALL_GET_ITER_VIRTUAL,
[GET_LEN] = _TAIL_CALL_GET_LEN,
[IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM,
[IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME,
@ -1008,9 +1014,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,
[216] = _TAIL_CALL_UNKNOWN_OPCODE,
[217] = _TAIL_CALL_UNKNOWN_OPCODE,
[218] = _TAIL_CALL_UNKNOWN_OPCODE,
[219] = _TAIL_CALL_UNKNOWN_OPCODE,
@ -1120,10 +1123,13 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD,
[FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD,
[FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD,
[FOR_ITER_VIRTUAL] = _TAIL_CALL_TRACE_RECORD,
[GET_AITER] = _TAIL_CALL_TRACE_RECORD,
[GET_ANEXT] = _TAIL_CALL_TRACE_RECORD,
[GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD,
[GET_ITER] = _TAIL_CALL_TRACE_RECORD,
[GET_ITER_SELF] = _TAIL_CALL_TRACE_RECORD,
[GET_ITER_VIRTUAL] = _TAIL_CALL_TRACE_RECORD,
[GET_LEN] = _TAIL_CALL_TRACE_RECORD,
[IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD,
[IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD,
@ -1266,9 +1272,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,
[216] = _TAIL_CALL_UNKNOWN_OPCODE,
[217] = _TAIL_CALL_UNKNOWN_OPCODE,
[218] = _TAIL_CALL_UNKNOWN_OPCODE,
[219] = _TAIL_CALL_UNKNOWN_OPCODE,

2
Python/optimizer.c
View file

@ -496,8 +496,10 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = {
[_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST,
[_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE,
[_FOR_ITER] = _FOR_ITER_TIER_TWO,
[_FOR_ITER_VIRTUAL] = _FOR_ITER_VIRTUAL_TIER_TWO,
[_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO,
[_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK,
[_LOAD_BYTECODE] = _NOP,
};
static const uint8_t

77
Python/optimizer_bytecodes.c
View file

@ -1396,9 +1396,46 @@ dummy_func(void) {
}
op(_GET_ITER, (iterable -- iter, index_or_null)) {
if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
bool is_coro = false;
bool is_trad = false; // has `tp_iter` slot
bool definite = true;
PyTypeObject *tp = sym_get_type(iterable);
if (tp == NULL) {
definite = false;
tp = sym_get_probable_type(iterable);
}
if (oparg == GET_ITER_YIELD_FROM_NO_CHECK) {
if (tp == &PyCoro_Type) {
if (!definite) {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
ADD_OP(_PUSH_NULL, 0, 0);
is_coro = true;
}
}
if (tp != NULL &&
tp->_tp_iteritem == NULL &&
tp->tp_iter != NULL &&
tp->tp_iter != PyObject_SelfIter &&
tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE
) {
assert(tp != &PyCoro_Type);
is_trad = true;
if (!definite) {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
ADD_OP(_GET_ITER_TRAD, 0, 0);
}
if (is_coro) {
assert(!is_trad);
iter = iterable;
index_or_null = sym_new_not_null(ctx);
index_or_null = sym_new_null(ctx);
}
else if (is_trad) {
iter = sym_new_not_null(ctx);
index_or_null = sym_new_null(ctx);
}
else {
iter = sym_new_not_null(ctx);
@ -1406,6 +1443,42 @@ dummy_func(void) {
}
}
op(_GUARD_ITERATOR, (iterable -- iterable)) {
bool definite = true;
PyTypeObject *tp = sym_get_type(iterable);
if (tp == NULL) {
definite = false;
tp = sym_get_probable_type(iterable);
}
if (tp != NULL && tp->tp_iter == PyObject_SelfIter) {
if (definite) {
ADD_OP(_NOP, 0, 0);
}
else {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
}
}
op(_GUARD_ITER_VIRTUAL, (iterable -- iterable)) {
bool definite = true;
PyTypeObject *tp = sym_get_type(iterable);
if (tp == NULL) {
definite = false;
tp = sym_get_probable_type(iterable);
}
if (tp != NULL && tp->_tp_iteritem != NULL) {
if (definite) {
ADD_OP(_NOP, 0, 0);
}
else {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
}
}
op(_FOR_ITER_GEN_FRAME, (iter, unused -- iter, unused, gen_frame)) {
_Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, iter, NULL, 0);
if (new_frame == NULL) {

126
Python/optimizer_cases.c.h generated
View file

@ -2572,6 +2572,10 @@
break;
}
case _GUARD_TYPE: {
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
break;
}
@ -3502,9 +3506,46 @@
JitOptRef iter;
JitOptRef index_or_null;
iterable = stack_pointer[-1];
if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) {
bool is_coro = false;
bool is_trad = false;
bool definite = true;
PyTypeObject *tp = sym_get_type(iterable);
if (tp == NULL) {
definite = false;
tp = sym_get_probable_type(iterable);
}
if (oparg == GET_ITER_YIELD_FROM_NO_CHECK) {
if (tp == &PyCoro_Type) {
if (!definite) {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
ADD_OP(_PUSH_NULL, 0, 0);
is_coro = true;
}
}
if (tp != NULL &&
tp->_tp_iteritem == NULL &&
tp->tp_iter != NULL &&
tp->tp_iter != PyObject_SelfIter &&
tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE
) {
assert(tp != &PyCoro_Type);
is_trad = true;
if (!definite) {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
ADD_OP(_GET_ITER_TRAD, 0, 0);
}
if (is_coro) {
assert(!is_trad);
iter = iterable;
index_or_null = sym_new_not_null(ctx);
index_or_null = sym_new_null(ctx);
}
else if (is_trad) {
iter = sym_new_not_null(ctx);
index_or_null = sym_new_null(ctx);
}
else {
iter = sym_new_not_null(ctx);
@ -3518,6 +3559,71 @@
break;
}
case _GUARD_ITERATOR: {
JitOptRef iterable;
iterable = stack_pointer[-1];
bool definite = true;
PyTypeObject *tp = sym_get_type(iterable);
if (tp == NULL) {
definite = false;
tp = sym_get_probable_type(iterable);
}
if (tp != NULL && tp->tp_iter == PyObject_SelfIter) {
if (definite) {
ADD_OP(_NOP, 0, 0);
}
else {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
}
break;
}
case _GUARD_ITER_VIRTUAL: {
JitOptRef iterable;
iterable = stack_pointer[-1];
bool definite = true;
PyTypeObject *tp = sym_get_type(iterable);
if (tp == NULL) {
definite = false;
tp = sym_get_probable_type(iterable);
}
if (tp != NULL && tp->_tp_iteritem != NULL) {
if (definite) {
ADD_OP(_NOP, 0, 0);
}
else {
ADD_OP(_GUARD_TYPE, 0, (uintptr_t)tp);
sym_set_type(iterable, tp);
}
}
break;
}
case _PUSH_TAGGED_ZERO: {
JitOptRef zero;
zero = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[0] = zero;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _GET_ITER_TRAD: {
JitOptRef iter;
JitOptRef index_or_null;
iter = sym_new_not_null(ctx);
index_or_null = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-1] = iter;
stack_pointer[0] = index_or_null;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
/* _FOR_ITER is not a viable micro-op for tier 2 */
case _FOR_ITER_TIER_TWO: {
@ -3530,6 +3636,22 @@
break;
}
case _GUARD_NOS_ITER_VIRTUAL: {
break;
}
/* _FOR_ITER_VIRTUAL is not a viable micro-op for tier 2 */
case _FOR_ITER_VIRTUAL_TIER_TWO: {
JitOptRef next;
next = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[0] = next;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */
case _ITER_CHECK_LIST: {

3
Python/record_functions.c.h generated
View file

@ -122,6 +122,9 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = {
[STORE_ATTR_INSTANCE_VALUE] = {1, {_RECORD_TOS_TYPE_INDEX}},
[STORE_ATTR_WITH_HINT] = {1, {_RECORD_TOS_TYPE_INDEX}},
[STORE_ATTR_SLOT] = {1, {_RECORD_TOS_TYPE_INDEX}},
[GET_ITER] = {1, {_RECORD_TOS_TYPE_INDEX}},
[GET_ITER_SELF] = {1, {_RECORD_TOS_TYPE_INDEX}},
[GET_ITER_VIRTUAL] = {1, {_RECORD_TOS_TYPE_INDEX}},
[FOR_ITER_GEN] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}},
[LOAD_SPECIAL] = {1, {_RECORD_TOS_TYPE_INDEX}},
[LOAD_ATTR_METHOD_WITH_VALUES] = {1, {_RECORD_TOS_TYPE_INDEX}},

45
Python/specialize.c
View file

@ -2715,20 +2715,24 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT
}
}
else {
if (tp == &PyList_Type) {
#ifdef Py_GIL_DISABLED
// Only specialize for lists owned by this thread or shared
if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
goto failure;
if (tp->_tp_iteritem != NULL) {
if (tp == &PyList_Type) {
#ifdef Py_GIL_DISABLED
// Only specialize for lists owned by this thread or shared
if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) {
goto failure;
}
#endif
specialize(instr, FOR_ITER_LIST);
return;
}
else if (tp == &PyTuple_Type) {
specialize(instr, FOR_ITER_TUPLE);
return;
}
#endif
specialize(instr, FOR_ITER_LIST);
return;
}
else if (tp == &PyTuple_Type) {
specialize(instr, FOR_ITER_TUPLE);
return;
}
specialize(instr, FOR_ITER_VIRTUAL);
return;
}
failure:
SPECIALIZATION_FAIL(FOR_ITER,
@ -2923,6 +2927,23 @@ _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr)
return;
}
void
_Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr)
{
assert(ENABLE_SPECIALIZATION);
PyTypeObject *tp = PyStackRef_TYPE(iterable);
if (tp->_tp_iteritem != NULL) {
specialize(instr, GET_ITER_VIRTUAL);
return;
}
if (tp->tp_iter == PyObject_SelfIter) {
specialize(instr, GET_ITER_SELF);
return;
}
SPECIALIZATION_FAIL(GET_ITER,
tp == &PyCoro_Type ? SPEC_FAIL_ITER_COROUTINE : SPEC_FAIL_OTHER);
unspecialize(instr);
}
void
_Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame)