Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-06-05 01:10:53 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-143732: SEND specialization (GH-148963)

Browse source 
* SEND specialization. Adds 2 new specialized instructions:

* SEND_VIRTUAL: for sends to virtual iterators e.g lists and tuples
* SEND_ASYNC_GEN: for sends to async generators

Tweak FOR_ITER_VIRTUAL so that SEND_VIRTUAL and FOR_ITER_VIRTUAL use equivalent guards
This commit is contained in:
Mark Shannon 2026-05-05 15:19:16 +01:00 • committed by GitHub
parent ffb543d32f
commit 70bd1c2dd2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 2662 additions and 1548 deletions
Download patch file Download diff file Expand all files Collapse all files

156
Python/bytecodes.c
View file

@ -1649,6 +1649,8 @@ dummy_func(
family(SEND, INLINE_CACHE_ENTRIES_SEND) = {
SEND_GEN,
SEND_VIRTUAL,
SEND_ASYNC_GEN,
};
specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused, unused -- receiver, unused, unused)) {
@ -1663,9 +1665,8 @@ dummy_func(
#endif /* ENABLE_SPECIALIZATION */
}
op(_SEND, (receiver, null_or_index, v -- receiver, null_or_index, retval)) {
tier1 op(_SEND, (receiver, null_or_index, v -- receiver, null_or_index, retval)) {
PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
PyObject *retval_o;
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
if (!IS_PEP523_HOOKED(tstate) &&
(Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
@ -1684,7 +1685,7 @@ dummy_func(
gen_frame->previous = frame;
DISPATCH_INLINED(gen_frame);
}
if (!PyStackRef_IsNull(null_or_index)) {
if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) {
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index);
if (!PyStackRef_IsValid(item)) {
if (PyStackRef_IsError(item)) {
@ -1694,34 +1695,20 @@ dummy_func(
DISPATCH();
}
retval = item;
DEAD(v);
}
else {
if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) {
retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
PyObject *v_o = PyStackRef_AsPyObjectBorrow(v);
PySendResultPair res = _PyIter_Send(receiver_o, v_o);
if (res.kind == PYGEN_ERROR) {
ERROR_NO_POP();
}
else {
retval_o = PyObject_CallMethodOneArg(receiver_o,
&_Py_ID(send),
PyStackRef_AsPyObjectBorrow(v));
PyStackRef_CLOSE(v);
retval = PyStackRef_FromPyObjectSteal(res.object);
if (res.kind == PYGEN_RETURN) {
JUMPBY(oparg);
}
if (retval_o == NULL) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
if (matches) {
_PyEval_MonitorRaise(tstate, frame, this_instr);
}
int err = _PyGen_FetchStopIterationValue(&retval_o);
if (err == 0) {
assert(retval_o != NULL);
JUMPBY(oparg);
}
else {
PyStackRef_CLOSE(v);
ERROR_IF(true);
}
}
retval = PyStackRef_FromPyObjectSteal(retval_o);
}
PyStackRef_CLOSE(v);
}
macro(SEND) = _SPECIALIZE_SEND + _SEND;
@ -1749,6 +1736,112 @@ dummy_func(
_SEND_GEN_FRAME +
_PUSH_FRAME;
op(_GUARD_TOS_IS_NONE, (val -- val)) {
EXIT_IF(!PyStackRef_IsNone(val));
}
macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) {
EXIT_IF(PyStackRef_IsNull(nos));
}
replaced op(_SEND_VIRTUAL, (iter, null_or_index, none -- iter, null_or_index, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
ERROR_NO_POP();
}
next = none;
DEAD(none);
JUMPBY(oparg);
DISPATCH();
}
DEAD(none);
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
}
op(_SEND_VIRTUAL_TIER_TWO, (iter, null_or_index, none -- iter, null_or_index, next)) {
assert(PyStackRef_IsNone(none));
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
ERROR_NO_POP();
}
next = none;
DEAD(none);
EXIT_IF(true);
}
DEAD(none);
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
}
macro(SEND_VIRTUAL) =
unused/1 +
_GUARD_TOS_IS_NONE +
_GUARD_NOS_NOT_NULL +
_SEND_VIRTUAL;
op(_GUARD_3OS_ASYNC_GEN_ASEND, (iter, null_or_index, value -- iter, null_or_index, value)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(!PyAsyncGenASend_CheckExact(iter_o));
}
replaced op(_SEND_ASYNC_GEN, (iter, null_in, v -- asend, null_out, retval)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
assert(PyAsyncGenASend_CheckExact(iter_o));
PyObject *val = PyStackRef_AsPyObjectBorrow(v);
PyObject *retval_o;
PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o);
if (what == PYGEN_ERROR) {
ERROR_NO_POP();
}
PyStackRef_CLOSE(v);
asend = iter;
DEAD(iter);
null_out = null_in;
DEAD(null_in);
retval = PyStackRef_FromPyObjectSteal(retval_o);
if (what == PYGEN_RETURN) {
JUMPBY(oparg);
}
}
macro(SEND_ASYNC_GEN) =
unused/1 +
_GUARD_3OS_ASYNC_GEN_ASEND +
_SEND_ASYNC_GEN;
tier2 op(_SEND_ASYNC_GEN_TIER_TWO, (iter, null_in, v -- asend, null_out, retval)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
assert(PyAsyncGenASend_CheckExact(iter_o));
PyObject *val = PyStackRef_AsPyObjectBorrow(v);
PyObject *retval_o;
PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o);
if (what == PYGEN_ERROR) {
ERROR_NO_POP();
}
PyStackRef_CLOSE(v);
asend = iter;
DEAD(iter);
null_out = null_in;
DEAD(null_in);
retval = PyStackRef_FromPyObjectSteal(retval_o);
EXIT_IF(what == PYGEN_RETURN);
}
op(_YIELD_VALUE, (retval -- value)) {
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
@ -3803,6 +3896,10 @@ dummy_func(
EXIT_IF(Py_TYPE(iter_o)->_tp_iteritem == NULL);
}
op(_GUARD_TOS_NOT_NULL, (null_or_index -- null_or_index)) {
EXIT_IF(PyStackRef_IsNull(null_or_index));
}
replaced op(_FOR_ITER_VIRTUAL, (iter, null_or_index -- iter, null_or_index, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
@ -3823,7 +3920,7 @@ dummy_func(
macro(FOR_ITER_VIRTUAL) =
unused/1 + // Skip over the counter
_GUARD_NOS_ITER_VIRTUAL +
_GUARD_TOS_NOT_NULL +
_FOR_ITER_VIRTUAL;
op(_FOR_ITER_VIRTUAL_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) {
@ -4606,11 +4703,6 @@ dummy_func(
EXIT_IF(!PyStackRef_IsNull(null));
}
op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
EXIT_IF(o == NULL);
}
op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) {
EXIT_IF(!PyStackRef_IsNull(null));
}

3
Python/ceval_macros.h
View file

@ -640,3 +640,6 @@ gen_try_set_executing(PyGenObject *gen)
(PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \
}
#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \
Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX))

2
Python/codegen.c
View file

@ -2156,7 +2156,7 @@ codegen_for(compiler *c, stmt_ty s)
USE_LABEL(c, cleanup);
/* It is important for instrumentation that the `END_FOR` comes first.
* Iteration over a generator will jump to the first of these instructions,
* but a non-generator will jump to a later instruction.
* but a non-generator will jump to the second instruction.
*/
ADDOP(c, NO_LOCATION, END_FOR);
ADDOP(c, NO_LOCATION, POP_ITER);

681
Python/executor_cases.c.h generated
View file

@ -8768,8 +8768,6 @@
break;
}
/* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _SEND_GEN_FRAME_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
@ -8816,6 +8814,511 @@
break;
}
case _GUARD_TOS_IS_NONE_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef val;
val = stack_pointer[-1];
if (!PyStackRef_IsNone(val)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = val;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TOS_IS_NONE_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef val;
_PyStackRef _stack_item_0 = _tos_cache0;
val = _stack_item_0;
if (!PyStackRef_IsNone(val)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = val;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = val;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TOS_IS_NONE_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef val;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
val = _stack_item_1;
if (!PyStackRef_IsNone(val)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = val;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = val;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TOS_IS_NONE_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef val;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
val = _stack_item_2;
if (!PyStackRef_IsNone(val)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = val;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = val;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r02: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
nos = stack_pointer[-2];
if (PyStackRef_IsNull(nos)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r12: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
_PyStackRef _stack_item_0 = _tos_cache0;
nos = stack_pointer[-1];
if (PyStackRef_IsNull(nos)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = _stack_item_0;
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
nos = _stack_item_0;
if (PyStackRef_IsNull(nos)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = _stack_item_1;
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
nos = _stack_item_1;
if (PyStackRef_IsNull(nos)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = nos;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = nos;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
/* _SEND_VIRTUAL is not a viable micro-op for tier 2 because it is replaced */
case _SEND_VIRTUAL_TIER_TWO_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef none;
_PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
none = stack_pointer[-1];
null_or_index = stack_pointer[-2];
iter = stack_pointer[-3];
assert(PyStackRef_IsNone(none));
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
next = none;
if (true) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
_tos_cache2 = next;
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _SEND_VIRTUAL_TIER_TWO_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef none;
_PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
_PyStackRef _stack_item_0 = _tos_cache0;
none = _stack_item_0;
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
assert(PyStackRef_IsNone(none));
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
stack_pointer[0] = none;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
next = none;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = stack_pointer[0];
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
_tos_cache2 = next;
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _SEND_VIRTUAL_TIER_TWO_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef none;
_PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
none = _stack_item_1;
null_or_index = _stack_item_0;
iter = stack_pointer[-1];
assert(PyStackRef_IsNone(none));
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
stack_pointer[0] = null_or_index;
stack_pointer[1] = none;
stack_pointer += 2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
next = none;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = stack_pointer[1];
_tos_cache0 = null_or_index;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
_tos_cache2 = next;
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _SEND_VIRTUAL_TIER_TWO_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef none;
_PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
none = _stack_item_2;
null_or_index = _stack_item_1;
iter = _stack_item_0;
assert(PyStackRef_IsNone(none));
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
stack_pointer[0] = iter;
stack_pointer[1] = null_or_index;
stack_pointer[2] = none;
stack_pointer += 3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
next = none;
if (true) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = stack_pointer[2];
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
_tos_cache2 = next;
_tos_cache1 = null_or_index;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_3OS_ASYNC_GEN_ASEND_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
iter = stack_pointer[-3];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyAsyncGenASend_CheckExact(iter_o)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = stack_pointer[-1];
_tos_cache1 = stack_pointer[-2];
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_3OS_ASYNC_GEN_ASEND_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
_PyStackRef _stack_item_0 = _tos_cache0;
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyAsyncGenASend_CheckExact(iter_o)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_0;
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_3OS_ASYNC_GEN_ASEND_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
iter = stack_pointer[-1];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyAsyncGenASend_CheckExact(iter_o)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_1;
_tos_cache1 = _stack_item_0;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_3OS_ASYNC_GEN_ASEND_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef iter;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
iter = _stack_item_0;
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyAsyncGenASend_CheckExact(iter_o)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = iter;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
/* _SEND_ASYNC_GEN is not a viable micro-op for tier 2 because it is replaced */
case _SEND_ASYNC_GEN_TIER_TWO_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef v;
_PyStackRef null_in;
_PyStackRef iter;
_PyStackRef asend;
_PyStackRef null_out;
_PyStackRef retval;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
v = _stack_item_2;
null_in = _stack_item_1;
iter = _stack_item_0;
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
assert(PyAsyncGenASend_CheckExact(iter_o));
PyObject *val = PyStackRef_AsPyObjectBorrow(v);
PyObject *retval_o;
stack_pointer[0] = iter;
stack_pointer[1] = null_in;
stack_pointer[2] = v;
stack_pointer += 3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (what == PYGEN_ERROR) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(v);
stack_pointer = _PyFrame_GetStackPointer(frame);
asend = iter;
null_out = null_in;
retval = PyStackRef_FromPyObjectSteal(retval_o);
if (what == PYGEN_RETURN) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = stack_pointer[-1];
_tos_cache1 = stack_pointer[-2];
_tos_cache0 = stack_pointer[-3];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = retval;
_tos_cache1 = null_out;
_tos_cache0 = asend;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _YIELD_VALUE_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
@ -10320,9 +10823,7 @@
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (none_val == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (matches &&
(Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
{
@ -10535,9 +11036,7 @@
int err = PyDict_Update(dict_o, update_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (matches) {
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *exc = _PyErr_GetRaisedException(tstate);
@ -14309,6 +14808,87 @@
break;
}
case _GUARD_TOS_NOT_NULL_r01: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef null_or_index;
null_or_index = stack_pointer[-1];
if (PyStackRef_IsNull(null_or_index)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = null_or_index;
SET_CURRENT_CACHED_VALUES(1);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TOS_NOT_NULL_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef null_or_index;
_PyStackRef _stack_item_0 = _tos_cache0;
null_or_index = _stack_item_0;
if (PyStackRef_IsNull(null_or_index)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = null_or_index;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache0 = null_or_index;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TOS_NOT_NULL_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef null_or_index;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
null_or_index = _stack_item_1;
if (PyStackRef_IsNull(null_or_index)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = null_or_index;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = null_or_index;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_TOS_NOT_NULL_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef null_or_index;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
null_or_index = _stack_item_2;
if (PyStackRef_IsNull(null_or_index)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = null_or_index;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = null_or_index;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
/* _FOR_ITER_VIRTUAL is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_VIRTUAL_TIER_TWO_r23: {
@ -16934,95 +17514,6 @@
break;
}
case _GUARD_NOS_NOT_NULL_r02: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (o == NULL) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r12: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
_PyStackRef _stack_item_0 = _tos_cache0;
nos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (o == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = _stack_item_0;
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r22: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
nos = _stack_item_0;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (o == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache1 = _stack_item_1;
_tos_cache0 = nos;
SET_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_NOT_NULL_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef nos;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
nos = _stack_item_1;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (o == NULL) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = nos;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = nos;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_THIRD_NULL_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());

199
Python/generated_cases.c.h generated
View file

@ -3782,8 +3782,7 @@
// _GUARD_NOS_NOT_NULL
{
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (o == NULL) {
if (PyStackRef_IsNull(nos)) {
UPDATE_MISS_STATS(CALL);
assert(_PyOpcode_Deopt[opcode] == (CALL));
JUMP_TO_PREDICTED(CALL);
@ -5784,9 +5783,7 @@
int err = PyDict_Update(dict_o, update_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (matches) {
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *exc = _PyErr_GetRaisedException(tstate);
@ -6407,15 +6404,14 @@
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_VIRTUAL);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
_PyStackRef iter;
_PyStackRef null_or_index;
_PyStackRef iter;
_PyStackRef next;
/* Skip 1 cache entry */
// _GUARD_NOS_ITER_VIRTUAL
// _GUARD_TOS_NOT_NULL
{
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o)->_tp_iteritem == NULL) {
null_or_index = stack_pointer[-1];
if (PyStackRef_IsNull(null_or_index)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
@ -6423,7 +6419,7 @@
}
// _FOR_ITER_VIRTUAL
{
null_or_index = stack_pointer[-1];
iter = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -8237,9 +8233,7 @@
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (none_val == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (matches &&
(Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
{
@ -11114,7 +11108,6 @@
v = stack_pointer[-1];
null_or_index = stack_pointer[-2];
PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
PyObject *retval_o;
assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
if (!IS_PEP523_HOOKED(tstate) &&
(Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
@ -11133,7 +11126,7 @@
gen_frame->previous = frame;
DISPATCH_INLINED(gen_frame);
}
if (!PyStackRef_IsNull(null_or_index)) {
if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index);
stack_pointer = _PyFrame_GetStackPointer(frame);
@ -11148,51 +11141,89 @@
retval = item;
}
else {
if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyObject *v_o = PyStackRef_AsPyObjectBorrow(v);
_PyFrame_SetStackPointer(frame, stack_pointer);
PySendResultPair res = _PyIter_Send(receiver_o, v_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res.kind == PYGEN_ERROR) {
JUMP_TO_LABEL(error);
}
else {
_PyFrame_SetStackPointer(frame, stack_pointer);
retval_o = PyObject_CallMethodOneArg(receiver_o,
&_Py_ID(send),
PyStackRef_AsPyObjectBorrow(v));
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(v);
stack_pointer = _PyFrame_GetStackPointer(frame);
retval = PyStackRef_FromPyObjectSteal(res.object);
if (res.kind == PYGEN_RETURN) {
JUMPBY(oparg);
}
if (retval_o == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (matches) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyEval_MonitorRaise(tstate, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _PyGen_FetchStopIterationValue(&retval_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err == 0) {
assert(retval_o != NULL);
JUMPBY(oparg);
}
else {
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(v);
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_LABEL(error);
}
}
retval = PyStackRef_FromPyObjectSteal(retval_o);
stack_pointer += 1;
}
stack_pointer[-2] = null_or_index;
stack_pointer[-1] = retval;
}
stack_pointer[-2] = null_or_index;
stack_pointer[-1] = retval;
DISPATCH();
}
TARGET(SEND_ASYNC_GEN) {
#if _Py_TAIL_CALL_INTERP
int opcode = SEND_ASYNC_GEN;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_ASYNC_GEN);
static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
_PyStackRef iter;
_PyStackRef null_in;
_PyStackRef v;
_PyStackRef asend;
_PyStackRef null_out;
_PyStackRef retval;
/* Skip 1 cache entry */
// _GUARD_3OS_ASYNC_GEN_ASEND
{
iter = stack_pointer[-3];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (!PyAsyncGenASend_CheckExact(iter_o)) {
UPDATE_MISS_STATS(SEND);
assert(_PyOpcode_Deopt[opcode] == (SEND));
JUMP_TO_PREDICTED(SEND);
}
}
// _SEND_ASYNC_GEN
{
v = stack_pointer[-1];
null_in = stack_pointer[-2];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
assert(PyAsyncGenASend_CheckExact(iter_o));
PyObject *val = PyStackRef_AsPyObjectBorrow(v);
PyObject *retval_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (what == PYGEN_ERROR) {
JUMP_TO_LABEL(error);
}
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(v);
stack_pointer = _PyFrame_GetStackPointer(frame);
asend = iter;
null_out = null_in;
retval = PyStackRef_FromPyObjectSteal(retval_o);
if (what == PYGEN_RETURN) {
JUMPBY(oparg);
}
}
stack_pointer[-2] = asend;
stack_pointer[-1] = null_out;
stack_pointer[0] = retval;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
DISPATCH();
}
@ -11265,6 +11296,70 @@
DISPATCH();
}
TARGET(SEND_VIRTUAL) {
#if _Py_TAIL_CALL_INTERP
int opcode = SEND_VIRTUAL;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_VIRTUAL);
static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
_PyStackRef val;
_PyStackRef nos;
_PyStackRef iter;
_PyStackRef null_or_index;
_PyStackRef none;
_PyStackRef next;
/* Skip 1 cache entry */
// _GUARD_TOS_IS_NONE
{
val = stack_pointer[-1];
if (!PyStackRef_IsNone(val)) {
UPDATE_MISS_STATS(SEND);
assert(_PyOpcode_Deopt[opcode] == (SEND));
JUMP_TO_PREDICTED(SEND);
}
}
// _GUARD_NOS_NOT_NULL
{
nos = stack_pointer[-2];
if (PyStackRef_IsNull(nos)) {
UPDATE_MISS_STATS(SEND);
assert(_PyOpcode_Deopt[opcode] == (SEND));
JUMP_TO_PREDICTED(SEND);
}
}
// _SEND_VIRTUAL
{
none = val;
null_or_index = nos;
iter = stack_pointer[-3];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
Py_ssize_t index = PyStackRef_UntagInt(null_or_index);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
PyObject *next_o = next_index.object;
index = next_index.index;
if (next_o == NULL) {
if (index < 0) {
JUMP_TO_LABEL(error);
}
next = none;
JUMPBY(oparg);
DISPATCH();
}
next = PyStackRef_FromPyObjectSteal(next_o);
null_or_index = PyStackRef_TagInt(index);
}
stack_pointer[-2] = null_or_index;
stack_pointer[-1] = next;
DISPATCH();
}
TARGET(SETUP_ANNOTATIONS) {
#if _Py_TAIL_CALL_INTERP
int opcode = SETUP_ANNOTATIONS;

1
Python/jit.c
View file

@ -11,6 +11,7 @@
#include "pycore_floatobject.h"
#include "pycore_frame.h"
#include "pycore_function.h"
#include "pycore_genobject.h"
#include "pycore_import.h"
#include "pycore_interpframe.h"
#include "pycore_interpolation.h"

18
Python/opcode_targets.h generated
View file

@ -202,7 +202,9 @@ static void *opcode_targets_table[256] = {
&&TARGET_LOAD_SUPER_ATTR_METHOD,
&&TARGET_RESUME_CHECK,
&&TARGET_RESUME_CHECK_JIT,
&&TARGET_SEND_ASYNC_GEN,
&&TARGET_SEND_GEN,
&&TARGET_SEND_VIRTUAL,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_STORE_ATTR_SLOT,
&&TARGET_STORE_ATTR_WITH_HINT,
@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_POP_ITER,
&&TARGET_INSTRUMENTED_END_SEND,
@ -476,8 +476,8 @@ static void *opcode_tracing_targets_table[256] = {
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
@ -725,7 +725,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAM
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS);
@ -969,7 +971,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR,
[RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE,
[SEND] = _TAIL_CALL_SEND,
[SEND_ASYNC_GEN] = _TAIL_CALL_SEND_ASYNC_GEN,
[SEND_GEN] = _TAIL_CALL_SEND_GEN,
[SEND_VIRTUAL] = _TAIL_CALL_SEND_VIRTUAL,
[SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS,
[SET_ADD] = _TAIL_CALL_SET_ADD,
[SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE,
@ -1015,8 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[217] = _TAIL_CALL_UNKNOWN_OPCODE,
[218] = _TAIL_CALL_UNKNOWN_OPCODE,
[219] = _TAIL_CALL_UNKNOWN_OPCODE,
[220] = _TAIL_CALL_UNKNOWN_OPCODE,
[221] = _TAIL_CALL_UNKNOWN_OPCODE,
@ -1227,7 +1229,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD,
[RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD,
[SEND] = _TAIL_CALL_TRACE_RECORD,
[SEND_ASYNC_GEN] = _TAIL_CALL_TRACE_RECORD,
[SEND_GEN] = _TAIL_CALL_TRACE_RECORD,
[SEND_VIRTUAL] = _TAIL_CALL_TRACE_RECORD,
[SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD,
[SET_ADD] = _TAIL_CALL_TRACE_RECORD,
[SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD,
@ -1273,8 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[217] = _TAIL_CALL_UNKNOWN_OPCODE,
[218] = _TAIL_CALL_UNKNOWN_OPCODE,
[219] = _TAIL_CALL_UNKNOWN_OPCODE,
[220] = _TAIL_CALL_UNKNOWN_OPCODE,
[221] = _TAIL_CALL_UNKNOWN_OPCODE,

12
Python/optimizer.c
View file

@ -500,6 +500,8 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = {
[_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO,
[_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK,
[_LOAD_BYTECODE] = _NOP,
[_SEND_VIRTUAL] = _SEND_VIRTUAL_TIER_TWO,
[_SEND_ASYNC_GEN] = _SEND_ASYNC_GEN_TIER_TWO,
};
static const uint8_t
@ -1000,8 +1002,14 @@ _PyJit_translate_single_bytecode_to_trace(
else {
int extended_arg = orig_oparg > 255;
uint32_t jump_target = next_inst + orig_oparg + extended_arg;
assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR);
assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER);
/* Jump must be to an "END" either END_FOR or END_SEND */
assert((
_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR &&
_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER
)
||
_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_SEND
);
if (is_for_iter_test[uop]) {
target = jump_target + 1;
}

25
Python/optimizer_bytecodes.c
View file

@ -2701,7 +2701,32 @@ dummy_func(void) {
stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer);
}
op(_GUARD_TOS_NOT_NULL, (null_or_index -- null_or_index)) {
if (sym_is_not_null(null_or_index)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
sym_set_non_null(null_or_index);
}
}
op(_GUARD_3OS_ASYNC_GEN_ASEND, (iter, null_or_index, value -- iter, null_or_index, value)) {
if (sym_matches_type(iter, &_PyAsyncGenASend_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
sym_set_type(iter, &_PyAsyncGenASend_Type);
}
}
op(_GET_ANEXT, (aiter -- aiter, awaitable)) {
if (sym_matches_type(aiter, &PyAsyncGen_Type)) {
awaitable = sym_new_type(ctx, &_PyAsyncGenASend_Type);
}
else {
awaitable = sym_new_not_null(ctx);
}
}
// END BYTECODES //

87
Python/optimizer_cases.c.h generated
View file

@ -1860,8 +1860,15 @@
}
case _GET_ANEXT: {
JitOptRef aiter;
JitOptRef awaitable;
awaitable = sym_new_not_null(ctx);
aiter = stack_pointer[-1];
if (sym_matches_type(aiter, &PyAsyncGen_Type)) {
awaitable = sym_new_type(ctx, &_PyAsyncGenASend_Type);
}
else {
awaitable = sym_new_not_null(ctx);
}
CHECK_STACK_BOUNDS(1);
stack_pointer[0] = awaitable;
stack_pointer += 1;
@ -1876,8 +1883,6 @@
break;
}
/* _SEND is not a viable micro-op for tier 2 */
case _SEND_GEN_FRAME: {
JitOptRef v;
JitOptRef receiver;
@ -1896,6 +1901,58 @@
break;
}
case _GUARD_TOS_IS_NONE: {
break;
}
case _GUARD_NOS_NOT_NULL: {
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_is_not_null(nos)) {
ADD_OP(_NOP, 0, 0);
}
else {
sym_set_non_null(nos);
}
break;
}
/* _SEND_VIRTUAL is not a viable micro-op for tier 2 */
case _SEND_VIRTUAL_TIER_TWO: {
JitOptRef next;
next = sym_new_not_null(ctx);
stack_pointer[-1] = next;
break;
}
case _GUARD_3OS_ASYNC_GEN_ASEND: {
JitOptRef iter;
iter = stack_pointer[-3];
if (sym_matches_type(iter, &_PyAsyncGenASend_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
sym_set_type(iter, &_PyAsyncGenASend_Type);
}
break;
}
/* _SEND_ASYNC_GEN is not a viable micro-op for tier 2 */
case _SEND_ASYNC_GEN_TIER_TWO: {
JitOptRef asend;
JitOptRef null_out;
JitOptRef retval;
asend = sym_new_not_null(ctx);
null_out = sym_new_not_null(ctx);
retval = sym_new_not_null(ctx);
stack_pointer[-3] = asend;
stack_pointer[-2] = null_out;
stack_pointer[-1] = retval;
break;
}
case _YIELD_VALUE: {
JitOptRef retval;
JitOptRef value;
@ -3687,6 +3744,18 @@
break;
}
case _GUARD_TOS_NOT_NULL: {
JitOptRef null_or_index;
null_or_index = stack_pointer[-1];
if (sym_is_not_null(null_or_index)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
sym_set_non_null(null_or_index);
}
break;
}
/* _FOR_ITER_VIRTUAL is not a viable micro-op for tier 2 */
case _FOR_ITER_VIRTUAL_TIER_TWO: {
@ -4211,18 +4280,6 @@
break;
}
case _GUARD_NOS_NOT_NULL: {
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_is_not_null(nos)) {
ADD_OP(_NOP, 0, 0);
}
else {
sym_set_non_null(nos);
}
break;
}
case _GUARD_THIRD_NULL: {
JitOptRef null;
null = stack_pointer[-3];

6
Python/record_functions.c.h generated
View file

@ -137,6 +137,9 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = {
[STORE_SUBSCR_DICT] = {1, {_RECORD_NOS_TYPE_INDEX}},
[SEND] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}},
[SEND_GEN] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}},
[FOR_ITER] = {1, {_RECORD_NOS_INDEX}},
[SEND_VIRTUAL] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}},
[SEND_ASYNC_GEN] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}},
[STORE_ATTR] = {1, {_RECORD_TOS_TYPE_INDEX}},
[LOAD_SUPER_ATTR] = {1, {_RECORD_NOS_INDEX}},
[LOAD_SUPER_ATTR_METHOD] = {1, {_RECORD_NOS_INDEX}},
@ -155,7 +158,6 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = {
[GET_ITER] = {1, {_RECORD_TOS_TYPE_INDEX}},
[GET_ITER_SELF] = {1, {_RECORD_TOS_TYPE_INDEX}},
[GET_ITER_VIRTUAL] = {1, {_RECORD_TOS_TYPE_INDEX}},
[FOR_ITER] = {1, {_RECORD_NOS_INDEX}},
[FOR_ITER_VIRTUAL] = {1, {_RECORD_NOS_INDEX}},
[FOR_ITER_LIST] = {1, {_RECORD_NOS_INDEX}},
[FOR_ITER_TUPLE] = {1, {_RECORD_NOS_INDEX}},
@ -204,6 +206,7 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = {
[BINARY_OP_SUBSCR_GETITEM] = {1, 0, {0}},
[STORE_SUBSCR_DICT] = {1, 0, {0}},
[SEND_GEN] = {1, 0, {0}},
[FOR_ITER] = {1, 1, {0}},
[LOAD_SUPER_ATTR_METHOD] = {1, 0, {0}},
[LOAD_ATTR_INSTANCE_VALUE] = {1, 1, {0}},
[LOAD_ATTR_WITH_HINT] = {1, 1, {0}},
@ -218,7 +221,6 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = {
[GET_ITER] = {1, 0, {0}},
[GET_ITER_SELF] = {1, 0, {0}},
[GET_ITER_VIRTUAL] = {1, 0, {0}},
[FOR_ITER] = {1, 1, {0}},
[FOR_ITER_GEN] = {1, 1, {0}},
[LOAD_SPECIAL] = {1, 0, {0}},
[LOAD_ATTR_METHOD_WITH_VALUES] = {1, 1, {0}},

8
Python/specialize.c
View file

@ -2761,6 +2761,14 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr)
specialize(instr, SEND_GEN);
return;
}
if (tp->_tp_iteritem != NULL) {
specialize(instr, SEND_VIRTUAL);
return;
}
if (tp == &_PyAsyncGenASend_Type) {
specialize(instr, SEND_ASYNC_GEN);
return;
}
SPECIALIZATION_FAIL(SEND,
_PySpecialization_ClassifyIterator(receiver));
failure: