Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-09 18:11:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-131798: Remove JIT guards for dict, frozenset, list, set, and tuple (GH-132289)

Browse source
This commit is contained in:
Brandt Bucher 2025-04-09 14:32:21 -07:00 • committed by GitHub
parent d47584aae6
commit 20926c73b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 942 additions and 543 deletions
Download patch file Download diff file Expand all files Collapse all files

109
Python/bytecodes.c
View file

@ -508,9 +508,21 @@ dummy_func(
}
}
inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) {
op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
EXIT_IF(!PyList_CheckExact(o));
}
op(_GUARD_TOS_LIST, (tos -- tos)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
EXIT_IF(!PyList_CheckExact(o));
}
macro(TO_BOOL_LIST) = _GUARD_TOS_LIST + unused/1 + unused/2 + _TO_BOOL_LIST;
op(_TO_BOOL_LIST, (value -- res)) {
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
EXIT_IF(!PyList_CheckExact(value_o));
assert(PyList_CheckExact(value_o));
STAT_INC(TO_BOOL, hit);
res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
DECREF_INPUTS();
@ -524,8 +536,7 @@ dummy_func(
res = PyStackRef_False;
}
op(_GUARD_NOS_UNICODE, (nos, tos -- nos, tos)) {
(void)tos;
op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
EXIT_IF(!PyUnicode_CheckExact(o));
}
@ -587,8 +598,7 @@ dummy_func(
BINARY_OP_EXTEND,
};
op(_GUARD_NOS_INT, (left, right -- left, right)) {
(void)right;
op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
EXIT_IF(!PyLong_CheckExact(left_o));
}
@ -650,8 +660,7 @@ dummy_func(
macro(BINARY_OP_SUBTRACT_INT) =
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT;
op(_GUARD_NOS_FLOAT, (left, right -- left, right)) {
(void)right;
op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
EXIT_IF(!PyFloat_CheckExact(left_o));
}
@ -862,14 +871,14 @@ dummy_func(
macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE;
macro(BINARY_OP_SUBSCR_LIST_INT) =
_GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_LIST_INT;
_GUARD_TOS_INT + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_INT;
op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
assert(PyList_CheckExact(list));
// Deopt unless 0 <= sub < PyList_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
@ -913,15 +922,25 @@ dummy_func(
res = PyStackRef_FromPyObjectImmortal(res_o);
}
op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
EXIT_IF(!PyTuple_CheckExact(o));
}
op(_GUARD_TOS_TUPLE, (tos -- tos)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
EXIT_IF(!PyTuple_CheckExact(o));
}
macro(BINARY_OP_SUBSCR_TUPLE_INT) =
_GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT;
_GUARD_TOS_INT + _GUARD_NOS_TUPLE + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT;
op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
assert(PyLong_CheckExact(sub));
DEOPT_IF(!PyTuple_CheckExact(tuple));
assert(PyTuple_CheckExact(tuple));
// Deopt unless 0 <= sub < PyTuple_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
@ -935,11 +954,24 @@ dummy_func(
DECREF_INPUTS();
}
inst(BINARY_OP_SUBSCR_DICT, (unused/5, dict_st, sub_st -- res)) {
op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
EXIT_IF(!PyDict_CheckExact(o));
}
op(_GUARD_TOS_DICT, (tos -- tos)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
EXIT_IF(!PyDict_CheckExact(o));
}
macro(BINARY_OP_SUBSCR_DICT) =
_GUARD_NOS_DICT + unused/5 + _BINARY_OP_SUBSCR_DICT;
op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict));
assert(PyDict_CheckExact(dict));
STAT_INC(BINARY_OP, hit);
PyObject *res_o;
int rc = PyDict_GetItemRef(dict, sub, &res_o);
@ -1021,14 +1053,14 @@ dummy_func(
macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR;
macro(STORE_SUBSCR_LIST_INT) =
_GUARD_TOS_INT + unused/1 + _STORE_SUBSCR_LIST_INT;
_GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT;
op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
assert(PyList_CheckExact(list));
// Ensure nonnegative, zero-or-one-digit ints.
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
@ -1052,10 +1084,13 @@ dummy_func(
Py_DECREF(old_value);
}
inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub -- )) {
macro(STORE_SUBSCR_DICT) =
_GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT;
op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- )) {
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict));
assert(PyDict_CheckExact(dict));
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict,
PyStackRef_AsPyObjectSteal(sub),
@ -1502,10 +1537,13 @@ dummy_func(
macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE;
inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) {
macro(UNPACK_SEQUENCE_TWO_TUPLE) =
_GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TWO_TUPLE;
op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) {
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
DEOPT_IF(!PyTuple_CheckExact(seq_o));
assert(PyTuple_CheckExact(seq_o));
DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2);
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
@ -1513,9 +1551,12 @@ dummy_func(
PyStackRef_CLOSE(seq);
}
inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) {
macro(UNPACK_SEQUENCE_TUPLE) =
_GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TUPLE;
op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) {
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
DEOPT_IF(!PyTuple_CheckExact(seq_o));
assert(PyTuple_CheckExact(seq_o));
DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
@ -1525,9 +1566,12 @@ dummy_func(
DECREF_INPUTS();
}
inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) {
macro(UNPACK_SEQUENCE_LIST) =
_GUARD_TOS_LIST + unused/1 + _UNPACK_SEQUENCE_LIST;
op(_UNPACK_SEQUENCE_LIST, (seq -- values[oparg])) {
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
DEOPT_IF(!PyList_CheckExact(seq_o));
assert(PyList_CheckExact(seq_o));
DEOPT_IF(!LOCK_OBJECT(seq_o));
if (PyList_GET_SIZE(seq_o) != oparg) {
UNLOCK_OBJECT(seq_o);
@ -2686,11 +2730,18 @@ dummy_func(
macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP;
inst(CONTAINS_OP_SET, (unused/1, left, right -- b)) {
op(_GUARD_TOS_ANY_SET, (tos -- tos)) {
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
DEOPT_IF(!PyAnySet_CheckExact(o));
}
macro(CONTAINS_OP_SET) = _GUARD_TOS_ANY_SET + unused/1 + _CONTAINS_OP_SET;
op(_CONTAINS_OP_SET, (left, right -- b)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o)));
assert(PyAnySet_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
int res = _PySet_Contains((PySetObject *)right_o, left_o);
@ -2699,11 +2750,13 @@ dummy_func(
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) {
macro(CONTAINS_OP_DICT) = _GUARD_TOS_DICT + unused/1 + _CONTAINS_OP_DICT;
op(_CONTAINS_OP_DICT, (left, right -- b)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
DEOPT_IF(!PyDict_CheckExact(right_o));
assert(PyDict_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
int res = PyDict_Contains(right_o, left_o);
DECREF_INPUTS();

141
Python/executor_cases.c.h generated
View file

@ -719,15 +719,34 @@
break;
}
case _GUARD_NOS_LIST: {
_PyStackRef nos;
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyList_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TOS_LIST: {
_PyStackRef tos;
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyList_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _TO_BOOL_LIST: {
_PyStackRef value;
_PyStackRef res;
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyList_CheckExact(value_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyList_CheckExact(value_o));
STAT_INC(TO_BOOL, hit);
res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -754,11 +773,8 @@
}
case _GUARD_NOS_UNICODE: {
_PyStackRef tos;
_PyStackRef nos;
tos = stack_pointer[-1];
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
@ -841,11 +857,8 @@
}
case _GUARD_NOS_INT: {
_PyStackRef right;
_PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
@ -950,11 +963,8 @@
}
case _GUARD_NOS_FLOAT: {
_PyStackRef right;
_PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
@ -1273,10 +1283,7 @@
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyList_CheckExact(list));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -1357,6 +1364,28 @@
break;
}
case _GUARD_NOS_TUPLE: {
_PyStackRef nos;
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyTuple_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TOS_TUPLE: {
_PyStackRef tos;
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyTuple_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _BINARY_OP_SUBSCR_TUPLE_INT: {
_PyStackRef sub_st;
_PyStackRef tuple_st;
@ -1366,10 +1395,7 @@
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
assert(PyLong_CheckExact(sub));
if (!PyTuple_CheckExact(tuple)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyTuple_CheckExact(tuple));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -1395,6 +1421,28 @@
break;
}
case _GUARD_NOS_DICT: {
_PyStackRef nos;
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyDict_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TOS_DICT: {
_PyStackRef tos;
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyDict_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _BINARY_OP_SUBSCR_DICT: {
_PyStackRef sub_st;
_PyStackRef dict_st;
@ -1403,10 +1451,7 @@
dict_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyDict_CheckExact(dict));
STAT_INC(BINARY_OP, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -1570,10 +1615,7 @@
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyList_CheckExact(list));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -1614,10 +1656,7 @@
dict_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyDict_CheckExact(dict));
STAT_INC(STORE_SUBSCR, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict,
@ -2049,10 +2088,7 @@
seq = stack_pointer[-1];
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
if (!PyTuple_CheckExact(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyTuple_CheckExact(seq_o));
if (PyTuple_GET_SIZE(seq_o) != 2) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -2077,10 +2113,7 @@
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
if (!PyTuple_CheckExact(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyTuple_CheckExact(seq_o));
if (PyTuple_GET_SIZE(seq_o) != oparg) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -2105,10 +2138,7 @@
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
if (!PyList_CheckExact(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyList_CheckExact(seq_o));
if (!LOCK_OBJECT(seq_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -3682,6 +3712,17 @@
break;
}
case _GUARD_TOS_ANY_SET: {
_PyStackRef tos;
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyAnySet_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _CONTAINS_OP_SET: {
_PyStackRef right;
_PyStackRef left;
@ -3691,10 +3732,7 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyAnySet_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = _PySet_Contains((PySetObject *)right_o, left_o);
@ -3728,10 +3766,7 @@
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyDict_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyDict_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyDict_Contains(right_o, left_o);

577
Python/generated_cases.c.h generated
View file

@ -108,9 +108,7 @@
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -121,6 +119,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_ADD_FLOAT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyFloat_CheckExact(left_o));
@ -167,9 +166,7 @@
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -180,6 +177,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_ADD_INT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
@ -214,7 +212,6 @@
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
@ -230,9 +227,7 @@
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -243,7 +238,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_ADD_UNICODE
{
right = tos;
right = value;
left = nos;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
@ -341,7 +336,6 @@
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
// _GUARD_TOS_UNICODE
@ -356,9 +350,7 @@
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -369,7 +361,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_INPLACE_ADD_UNICODE
{
right = tos;
right = value;
left = nos;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
assert(PyUnicode_CheckExact(left_o));
@ -441,9 +433,7 @@
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -454,6 +444,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_MULTIPLY_FLOAT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyFloat_CheckExact(left_o));
@ -500,9 +491,7 @@
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -513,6 +502,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_MULTIPLY_INT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
@ -545,45 +535,55 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef nos;
_PyStackRef dict_st;
_PyStackRef sub_st;
_PyStackRef res;
// _GUARD_NOS_DICT
{
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyDict_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
sub_st = stack_pointer[-1];
dict_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
int rc = PyDict_GetItemRef(dict, sub, &res_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (rc == 0) {
// _BINARY_OP_SUBSCR_DICT
{
sub_st = stack_pointer[-1];
dict_st = nos;
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
assert(PyDict_CheckExact(dict));
STAT_INC(BINARY_OP, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_SetKeyError(sub);
int rc = PyDict_GetItemRef(dict, sub, &res_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (rc == 0) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_SetKeyError(sub);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = sub_st;
sub_st = PyStackRef_NULL;
stack_pointer[-1] = sub_st;
PyStackRef_CLOSE(tmp);
tmp = dict_st;
dict_st = PyStackRef_NULL;
stack_pointer[-2] = dict_st;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (rc <= 0) {
JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = sub_st;
sub_st = PyStackRef_NULL;
stack_pointer[-1] = sub_st;
PyStackRef_CLOSE(tmp);
tmp = dict_st;
dict_st = PyStackRef_NULL;
stack_pointer[-2] = dict_st;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (rc <= 0) {
JUMP_TO_LABEL(error);
}
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@ -685,6 +685,7 @@
INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef list_st;
_PyStackRef sub_st;
_PyStackRef res;
@ -698,19 +699,25 @@
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBSCR_LIST_INT
// _GUARD_NOS_LIST
{
sub_st = value;
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBSCR_LIST_INT
{
sub_st = value;
list_st = nos;
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
assert(PyList_CheckExact(list));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
@ -769,7 +776,6 @@
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef str_st;
_PyStackRef sub_st;
_PyStackRef res;
@ -785,9 +791,7 @@
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -798,7 +802,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_SUBSCR_STR_INT
{
sub_st = tos;
sub_st = value;
str_st = nos;
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
@ -849,6 +853,7 @@
INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tuple_st;
_PyStackRef sub_st;
_PyStackRef res;
@ -862,19 +867,25 @@
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBSCR_TUPLE_INT
// _GUARD_NOS_TUPLE
{
sub_st = value;
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
assert(PyLong_CheckExact(sub));
if (!PyTuple_CheckExact(tuple)) {
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyTuple_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
// _BINARY_OP_SUBSCR_TUPLE_INT
{
sub_st = value;
tuple_st = nos;
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
assert(PyLong_CheckExact(sub));
assert(PyTuple_CheckExact(tuple));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
@ -930,9 +941,7 @@
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -943,6 +952,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_SUBTRACT_FLOAT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyFloat_CheckExact(left_o));
@ -989,9 +999,7 @@
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
@ -1002,6 +1010,7 @@
/* Skip 5 cache entries */
// _BINARY_OP_SUBTRACT_INT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyLong_CheckExact(left_o));
@ -4546,9 +4555,7 @@
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
@ -4559,6 +4566,7 @@
/* Skip 1 cache entry */
// _COMPARE_OP_FLOAT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
@ -4602,9 +4610,7 @@
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
@ -4615,6 +4621,7 @@
/* Skip 1 cache entry */
// _COMPARE_OP_INT
{
right = value;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!_PyLong_IsCompact((PyLongObject *)left_o)) {
@ -4656,7 +4663,6 @@
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
@ -4672,9 +4678,7 @@
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(COMPARE_OP);
@ -4685,7 +4689,7 @@
/* Skip 1 cache entry */
// _COMPARE_OP_STR
{
right = tos;
right = value;
left = nos;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
@ -4776,37 +4780,47 @@
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_DICT);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
_PyStackRef b;
// _GUARD_TOS_DICT
{
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyDict_CheckExact(o)) {
UPDATE_MISS_STATS(CONTAINS_OP);
assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
JUMP_TO_PREDICTED(CONTAINS_OP);
}
}
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyDict_CheckExact(right_o)) {
UPDATE_MISS_STATS(CONTAINS_OP);
assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
JUMP_TO_PREDICTED(CONTAINS_OP);
// _CONTAINS_OP_DICT
{
right = tos;
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyDict_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyDict_Contains(right_o, left_o);
_PyStackRef tmp = right;
right = PyStackRef_NULL;
stack_pointer[-1] = right;
PyStackRef_CLOSE(tmp);
tmp = left;
left = PyStackRef_NULL;
stack_pointer[-2] = left;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (res < 0) {
JUMP_TO_LABEL(error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = PyDict_Contains(right_o, left_o);
_PyStackRef tmp = right;
right = PyStackRef_NULL;
stack_pointer[-1] = right;
PyStackRef_CLOSE(tmp);
tmp = left;
left = PyStackRef_NULL;
stack_pointer[-2] = left;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (res < 0) {
JUMP_TO_LABEL(error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = b;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@ -4824,37 +4838,47 @@
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_SET);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
_PyStackRef b;
// _GUARD_TOS_ANY_SET
{
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyAnySet_CheckExact(o)) {
UPDATE_MISS_STATS(CONTAINS_OP);
assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
JUMP_TO_PREDICTED(CONTAINS_OP);
}
}
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) {
UPDATE_MISS_STATS(CONTAINS_OP);
assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP));
JUMP_TO_PREDICTED(CONTAINS_OP);
// _CONTAINS_OP_SET
{
right = tos;
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyAnySet_CheckExact(right_o));
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = _PySet_Contains((PySetObject *)right_o, left_o);
_PyStackRef tmp = right;
right = PyStackRef_NULL;
stack_pointer[-1] = right;
PyStackRef_CLOSE(tmp);
tmp = left;
left = PyStackRef_NULL;
stack_pointer[-2] = left;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (res < 0) {
JUMP_TO_LABEL(error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
}
STAT_INC(CONTAINS_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int res = _PySet_Contains((PySetObject *)right_o, left_o);
_PyStackRef tmp = right;
right = PyStackRef_NULL;
stack_pointer[-1] = right;
PyStackRef_CLOSE(tmp);
tmp = left;
left = PyStackRef_NULL;
stack_pointer[-2] = left;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
if (res < 0) {
JUMP_TO_LABEL(error);
}
b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False;
stack_pointer[0] = b;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@ -11205,32 +11229,42 @@
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
_PyStackRef nos;
_PyStackRef value;
_PyStackRef dict_st;
_PyStackRef sub;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
dict_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
// _GUARD_NOS_DICT
{
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyDict_CheckExact(o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
}
STAT_INC(STORE_SUBSCR, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict,
PyStackRef_AsPyObjectSteal(sub),
PyStackRef_AsPyObjectSteal(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(dict_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
JUMP_TO_LABEL(error);
/* Skip 1 cache entry */
// _STORE_SUBSCR_DICT
{
sub = stack_pointer[-1];
dict_st = nos;
value = stack_pointer[-3];
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
assert(PyDict_CheckExact(dict));
STAT_INC(STORE_SUBSCR, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict,
PyStackRef_AsPyObjectSteal(sub),
PyStackRef_AsPyObjectSteal(value));
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(dict_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) {
JUMP_TO_LABEL(error);
}
}
DISPATCH();
}
@ -11247,6 +11281,7 @@
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef list_st;
_PyStackRef sub_st;
// _GUARD_TOS_INT
@ -11259,20 +11294,26 @@
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
}
/* Skip 1 cache entry */
// _STORE_SUBSCR_LIST_INT
// _GUARD_NOS_LIST
{
sub_st = value;
list_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
nos = stack_pointer[-2];
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
}
/* Skip 1 cache entry */
// _STORE_SUBSCR_LIST_INT
{
sub_st = value;
list_st = nos;
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
assert(PyList_CheckExact(list));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
@ -11498,25 +11539,35 @@
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
_PyStackRef tos;
_PyStackRef value;
_PyStackRef res;
// _GUARD_TOS_LIST
{
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
JUMP_TO_PREDICTED(TO_BOOL);
}
}
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyList_CheckExact(value_o)) {
UPDATE_MISS_STATS(TO_BOOL);
assert(_PyOpcode_Deopt[opcode] == (TO_BOOL));
JUMP_TO_PREDICTED(TO_BOOL);
// _TO_BOOL_LIST
{
value = tos;
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
assert(PyList_CheckExact(value_o));
STAT_INC(TO_BOOL, hit);
res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = value;
value = res;
stack_pointer[-1] = value;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
STAT_INC(TO_BOOL, hit);
res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = value;
value = res;
stack_pointer[-1] = value;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH();
}
@ -11760,41 +11811,51 @@
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
_PyStackRef tos;
_PyStackRef seq;
_PyStackRef *values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
if (!PyList_CheckExact(seq_o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
if (!LOCK_OBJECT(seq_o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
if (PyList_GET_SIZE(seq_o) != oparg) {
UNLOCK_OBJECT(seq_o);
if (true) {
// _GUARD_TOS_LIST
{
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyList_CheckExact(o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
*values++ = PyStackRef_FromPyObjectNew(items[i]);
/* Skip 1 cache entry */
// _UNPACK_SEQUENCE_LIST
{
seq = tos;
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
assert(PyList_CheckExact(seq_o));
if (!LOCK_OBJECT(seq_o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
if (PyList_GET_SIZE(seq_o) != oparg) {
UNLOCK_OBJECT(seq_o);
if (true) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
*values++ = PyStackRef_FromPyObjectNew(items[i]);
}
UNLOCK_OBJECT(seq_o);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
UNLOCK_OBJECT(seq_o);
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH();
}
@ -11809,32 +11870,42 @@
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
_PyStackRef tos;
_PyStackRef seq;
_PyStackRef *values;
// _GUARD_TOS_TUPLE
{
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyTuple_CheckExact(o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
}
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
if (!PyTuple_CheckExact(seq_o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
// _UNPACK_SEQUENCE_TUPLE
{
seq = tos;
values = &stack_pointer[-1];
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
assert(PyTuple_CheckExact(seq_o));
if (PyTuple_GET_SIZE(seq_o) != oparg) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
*values++ = PyStackRef_FromPyObjectNew(items[i]);
}
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
if (PyTuple_GET_SIZE(seq_o) != oparg) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq_o);
for (int i = oparg; --i >= 0; ) {
*values++ = PyStackRef_FromPyObjectNew(items[i]);
}
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH();
}
@ -11849,33 +11920,43 @@
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
_PyStackRef tos;
_PyStackRef seq;
_PyStackRef val1;
_PyStackRef val0;
// _GUARD_TOS_TUPLE
{
tos = stack_pointer[-1];
PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
if (!PyTuple_CheckExact(o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
}
/* Skip 1 cache entry */
seq = stack_pointer[-1];
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
if (!PyTuple_CheckExact(seq_o)) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
// _UNPACK_SEQUENCE_TWO_TUPLE
{
seq = tos;
assert(oparg == 2);
PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq);
assert(PyTuple_CheckExact(seq_o));
if (PyTuple_GET_SIZE(seq_o) != 2) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
if (PyTuple_GET_SIZE(seq_o) != 2) {
UPDATE_MISS_STATS(UNPACK_SEQUENCE);
assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE));
JUMP_TO_PREDICTED(UNPACK_SEQUENCE);
}
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0));
val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1));
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH();
}

53
Python/optimizer_bytecodes.c
View file

@ -396,7 +396,6 @@ dummy_func(void) {
op(_TO_BOOL_LIST, (value -- res)) {
int already_bool = optimize_to_bool(this_instr, ctx, value, &res);
if (!already_bool) {
sym_set_type(value, &PyList_Type);
res = sym_new_type(ctx, &PyBool_Type);
}
}
@ -927,7 +926,57 @@ dummy_func(void) {
op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) {
for (int i = 0; i < oparg; i++) {
values[i] = sym_tuple_getitem(ctx, seq, i);
values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1);
}
}
op(_GUARD_TOS_LIST, (tos -- tos)) {
if (sym_matches_type(tos, &PyList_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyList_Type);
}
op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyList_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyList_Type);
}
op(_GUARD_TOS_TUPLE, (tos -- tos)) {
if (sym_matches_type(tos, &PyTuple_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyTuple_Type);
}
op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyTuple_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyTuple_Type);
}
op(_GUARD_TOS_DICT, (tos -- tos)) {
if (sym_matches_type(tos, &PyDict_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyDict_Type);
}
op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyDict_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyDict_Type);
}
op(_GUARD_TOS_ANY_SET, (tos -- tos)) {
if (sym_matches_type(tos, &PySet_Type) ||
sym_matches_type(tos, &PyFrozenSet_Type))
{
REPLACE_OP(this_instr, _NOP, 0, 0);
}
}

74
Python/optimizer_cases.c.h generated
View file

@ -195,13 +195,32 @@
break;
}
case _GUARD_NOS_LIST: {
JitOptSymbol *nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyList_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyList_Type);
break;
}
case _GUARD_TOS_LIST: {
JitOptSymbol *tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyList_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyList_Type);
break;
}
case _TO_BOOL_LIST: {
JitOptSymbol *value;
JitOptSymbol *res;
value = stack_pointer[-1];
int already_bool = optimize_to_bool(this_instr, ctx, value, &res);
if (!already_bool) {
sym_set_type(value, &PyList_Type);
res = sym_new_type(ctx, &PyBool_Type);
}
stack_pointer[-1] = res;
@ -576,6 +595,26 @@
break;
}
case _GUARD_NOS_TUPLE: {
JitOptSymbol *nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyTuple_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyTuple_Type);
break;
}
case _GUARD_TOS_TUPLE: {
JitOptSymbol *tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyTuple_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyTuple_Type);
break;
}
case _BINARY_OP_SUBSCR_TUPLE_INT: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
@ -585,6 +624,26 @@
break;
}
case _GUARD_NOS_DICT: {
JitOptSymbol *nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyDict_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyDict_Type);
break;
}
case _GUARD_TOS_DICT: {
JitOptSymbol *tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyDict_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyDict_Type);
break;
}
case _BINARY_OP_SUBSCR_DICT: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
@ -794,7 +853,7 @@
seq = stack_pointer[-1];
values = &stack_pointer[-1];
for (int i = 0; i < oparg; i++) {
values[i] = sym_tuple_getitem(ctx, seq, i);
values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1);
}
stack_pointer += -1 + oparg;
assert(WITHIN_STACK_BOUNDS());
@ -1282,6 +1341,17 @@
break;
}
case _GUARD_TOS_ANY_SET: {
JitOptSymbol *tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PySet_Type) ||
sym_matches_type(tos, &PyFrozenSet_Type))
{
REPLACE_OP(this_instr, _NOP, 0, 0);
}
break;
}
case _CONTAINS_OP_SET: {
JitOptSymbol *res;
res = sym_new_type(ctx, &PyBool_Type);