Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-30 21:21:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-98831: Typed stack effects, and more instructions converted (#99764)

Browse source 
Stack effects can now have a type, e.g. `inst(X, (left, right -- jump/uint64_t)) { ... }`.

Instructions converted to the non-legacy format:

* COMPARE_OP
* COMPARE_OP_FLOAT_JUMP
* COMPARE_OP_INT_JUMP
* COMPARE_OP_STR_JUMP
* STORE_ATTR
* DELETE_ATTR
* STORE_GLOBAL
* STORE_ATTR_INSTANCE_VALUE
* STORE_ATTR_WITH_HINT
* STORE_ATTR_SLOT, and complete the store_attr family
* Complete the store_subscr family: STORE_SUBSCR{,DICT,LIST_INT}
  (STORE_SUBSCR was alread half converted,
  but wasn't using cache effects yet.)
* DELETE_SUBSCR
* PRINT_EXPR
* INTERPRETER_EXIT (a bit weird, ends in return)
* RETURN_VALUE
* GET_AITER (had to restructure it some)
  The original had mysterious `SET_TOP(NULL)` before `goto error`.
  I assume those just account for `obj` having been decref'ed,
  so I got rid of them in favor of the cleanup implied by `ERROR_IF()`.
* LIST_APPEND (a bit unhappy with it)
* SET_ADD (also a bit unhappy with it)

Various other improvements/refactorings as well.
This commit is contained in:
Guido van Rossum 2022-12-08 13:31:27 -08:00 • committed by GitHub
parent 35cc0ea736
commit c85be734d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 545 additions and 604 deletions
Download patch file Download diff file Expand all files Collapse all files

346
Python/bytecodes.c
View file

@ -81,8 +81,17 @@ do { \
// Dummy variables for stack effects.
static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub;
static PyObject *container, *start, *stop, *v, *lhs, *rhs;
static PyObject *list, *tuple, *dict;
static PyObject *exit_func, *lasti, *val;
static PyObject *list, *tuple, *dict, *owner;
static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter;
static size_t jump;
// Dummy variables for cache effects
static _Py_CODEUNIT when_to_jump_mask, invert, counter, index, hint;
static uint32_t type_version;
// Dummy opcode names for 'op' opcodes
#define _COMPARE_OP_FLOAT 1003
#define _COMPARE_OP_INT 1004
#define _COMPARE_OP_STR 1005
#define _JUMP_IF 1006
static PyObject *
dummy_func(
@ -205,7 +214,7 @@ dummy_func(
};
inst(BINARY_OP_MULTIPLY_INT, (left, right, unused/1 -- prod)) {
inst(BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- prod)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
@ -216,7 +225,7 @@ dummy_func(
ERROR_IF(prod == NULL, error);
}
inst(BINARY_OP_MULTIPLY_FLOAT, (left, right, unused/1 -- prod)) {
inst(BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- prod)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
@ -229,7 +238,7 @@ dummy_func(
ERROR_IF(prod == NULL, error);
}
inst(BINARY_OP_SUBTRACT_INT, (left, right, unused/1 -- sub)) {
inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
@ -240,7 +249,7 @@ dummy_func(
ERROR_IF(sub == NULL, error);
}
inst(BINARY_OP_SUBTRACT_FLOAT, (left, right, unused/1 -- sub)) {
inst(BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- sub)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
@ -252,7 +261,7 @@ dummy_func(
ERROR_IF(sub == NULL, error);
}
inst(BINARY_OP_ADD_UNICODE, (left, right, unused/1 -- res)) {
inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
@ -299,7 +308,7 @@ dummy_func(
JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1);
}
inst(BINARY_OP_ADD_FLOAT, (left, right, unused/1 -- sum)) {
inst(BINARY_OP_ADD_FLOAT, (unused/1, left, right -- sum)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
@ -312,7 +321,7 @@ dummy_func(
ERROR_IF(sum == NULL, error);
}
inst(BINARY_OP_ADD_INT, (left, right, unused/1 -- sum)) {
inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
@ -331,7 +340,7 @@ dummy_func(
BINARY_SUBSCR_TUPLE_INT,
};
inst(BINARY_SUBSCR, (container, sub, unused/4 -- res)) {
inst(BINARY_SUBSCR, (unused/4, container, sub -- res)) {
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
@ -377,7 +386,7 @@ dummy_func(
ERROR_IF(err, error);
}
inst(BINARY_SUBSCR_LIST_INT, (list, sub, unused/4 -- res)) {
inst(BINARY_SUBSCR_LIST_INT, (unused/4, list, sub -- res)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
@ -396,7 +405,7 @@ dummy_func(
Py_DECREF(list);
}
inst(BINARY_SUBSCR_TUPLE_INT, (tuple, sub, unused/4 -- res)) {
inst(BINARY_SUBSCR_TUPLE_INT, (unused/4, tuple, sub -- res)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
@ -415,7 +424,7 @@ dummy_func(
Py_DECREF(tuple);
}
inst(BINARY_SUBSCR_DICT, (dict, sub, unused/4 -- res)) {
inst(BINARY_SUBSCR_DICT, (unused/4, dict, sub -- res)) {
assert(cframe.use_tracing == 0);
DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
@ -426,14 +435,14 @@ dummy_func(
}
Py_DECREF(dict);
Py_DECREF(sub);
ERROR_IF(1, error);
ERROR_IF(true, error);
}
Py_INCREF(res); // Do this before DECREF'ing dict, sub
Py_DECREF(dict);
Py_DECREF(sub);
}
inst(BINARY_SUBSCR_GETITEM, (container, sub, unused/1, type_version/2, func_version/1 -- unused)) {
inst(BINARY_SUBSCR_GETITEM, (unused/1, type_version/2, func_version/1, container, sub -- unused)) {
PyTypeObject *tp = Py_TYPE(container);
DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR);
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
@ -457,52 +466,48 @@ dummy_func(
DISPATCH_INLINED(new_frame);
}
// stack effect: (__0 -- )
inst(LIST_APPEND) {
PyObject *v = POP();
PyObject *list = PEEK(oparg);
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
goto error;
// Alternative: (list, unused[oparg], v -- list, unused[oparg])
inst(LIST_APPEND, (v --)) {
PyObject *list = PEEK(oparg + 1); // +1 to account for v staying on stack
ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error);
PREDICT(JUMP_BACKWARD);
}
// stack effect: (__0 -- )
inst(SET_ADD) {
PyObject *v = POP();
PyObject *set = PEEK(oparg);
int err;
err = PySet_Add(set, v);
// Alternative: (set, unused[oparg], v -- set, unused[oparg])
inst(SET_ADD, (v --)) {
PyObject *set = PEEK(oparg + 1); // +1 to account for v staying on stack
int err = PySet_Add(set, v);
Py_DECREF(v);
if (err != 0)
goto error;
ERROR_IF(err, error);
PREDICT(JUMP_BACKWARD);
}
inst(STORE_SUBSCR, (v, container, sub -- )) {
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
family(store_subscr) = {
STORE_SUBSCR,
STORE_SUBSCR_DICT,
STORE_SUBSCR_LIST_INT,
};
inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) {
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_SUBSCR, deferred);
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
/* container[sub] = v */
int err = PyObject_SetItem(container, sub, v);
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
ERROR_IF(err != 0, error);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
ERROR_IF(err, error);
}
// stack effect: (__0, __1, __2 -- )
inst(STORE_SUBSCR_LIST_INT) {
inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {
assert(cframe.use_tracing == 0);
PyObject *sub = TOP();
PyObject *list = SECOND();
PyObject *value = THIRD();
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
@ -515,60 +520,42 @@ dummy_func(
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, value);
STACK_SHRINK(3);
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
}
// stack effect: (__0, __1, __2 -- )
inst(STORE_SUBSCR_DICT) {
inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) {
assert(cframe.use_tracing == 0);
PyObject *sub = TOP();
PyObject *dict = SECOND();
PyObject *value = THIRD();
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
STACK_SHRINK(3);
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
if (err != 0) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
ERROR_IF(err, error);
}
// stack effect: (__0, __1 -- )
inst(DELETE_SUBSCR) {
PyObject *sub = TOP();
PyObject *container = SECOND();
int err;
STACK_SHRINK(2);
inst(DELETE_SUBSCR, (container, sub --)) {
/* del container[sub] */
err = PyObject_DelItem(container, sub);
int err = PyObject_DelItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
if (err != 0)
goto error;
ERROR_IF(err, error);
}
// stack effect: (__0 -- )
inst(PRINT_EXPR) {
PyObject *value = POP();
inst(PRINT_EXPR, (value --)) {
PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook));
PyObject *res;
// Can't use ERROR_IF here.
if (hook == NULL) {
_PyErr_SetString(tstate, PyExc_RuntimeError,
"lost sys.displayhook");
Py_DECREF(value);
goto error;
ERROR_IF(true, error);
}
res = PyObject_CallOneArg(hook, value);
Py_DECREF(value);
if (res == NULL)
goto error;
ERROR_IF(res == NULL, error);
Py_DECREF(res);
}
@ -595,11 +582,10 @@ dummy_func(
goto error;
}
// stack effect: (__0 -- )
inst(INTERPRETER_EXIT) {
inst(INTERPRETER_EXIT, (retval --)) {
assert(frame == &entry_frame);
assert(_PyFrame_IsIncomplete(frame));
PyObject *retval = POP();
STACK_SHRINK(1); // Since we're not going to DISPATCH()
assert(EMPTY());
/* Restore previous cframe and return. */
tstate->cframe = cframe.previous;
@ -610,9 +596,8 @@ dummy_func(
return retval;
}
// stack effect: (__0 -- )
inst(RETURN_VALUE) {
PyObject *retval = POP();
inst(RETURN_VALUE, (retval --)) {
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
TRACE_FUNCTION_EXIT();
@ -627,48 +612,37 @@ dummy_func(
goto resume_frame;
}
// stack effect: ( -- )
inst(GET_AITER) {
inst(GET_AITER, (obj -- iter)) {
unaryfunc getter = NULL;
PyObject *iter = NULL;
PyObject *obj = TOP();
PyTypeObject *type = Py_TYPE(obj);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
if (getter != NULL) {
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) {
SET_TOP(NULL);
goto error;
}
}
else {
SET_TOP(NULL);
if (getter == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
goto error;
ERROR_IF(true, error);
}
iter = (*getter)(obj);
Py_DECREF(obj);
ERROR_IF(iter == NULL, error);
if (Py_TYPE(iter)->tp_as_async == NULL ||
Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
SET_TOP(NULL);
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
goto error;
ERROR_IF(true, error);
}
SET_TOP(iter);
}
// stack effect: ( -- __0)
@ -1119,53 +1093,43 @@ dummy_func(
Py_DECREF(seq);
}
// stack effect: (__0, __1 -- )
inst(STORE_ATTR) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
family(store_attr) = {
STORE_ATTR,
STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR_SLOT,
STORE_ATTR_WITH_HINT,
};
inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) {
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg);
next_instr--;
_Py_Specialize_StoreAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_ATTR, deferred);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
PyObject *name = GETITEM(names, oparg);
PyObject *owner = TOP();
PyObject *v = SECOND();
int err;
STACK_SHRINK(2);
err = PyObject_SetAttr(owner, name, v);
int err = PyObject_SetAttr(owner, name, v);
Py_DECREF(v);
Py_DECREF(owner);
if (err != 0) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
ERROR_IF(err, error);
}
// stack effect: (__0 -- )
inst(DELETE_ATTR) {
inst(DELETE_ATTR, (owner --)) {
PyObject *name = GETITEM(names, oparg);
PyObject *owner = POP();
int err;
err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
int err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
Py_DECREF(owner);
if (err != 0)
goto error;
ERROR_IF(err, error);
}
// stack effect: (__0 -- )
inst(STORE_GLOBAL) {
inst(STORE_GLOBAL, (v --)) {
PyObject *name = GETITEM(names, oparg);
PyObject *v = POP();
int err;
err = PyDict_SetItem(GLOBALS(), name, v);
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
if (err != 0)
goto error;
ERROR_IF(err, error);
}
inst(DELETE_GLOBAL, (--)) {
@ -1954,22 +1918,15 @@ dummy_func(
DISPATCH_INLINED(new_frame);
}
// stack effect: (__0, __1 -- )
inst(STORE_ATTR_INSTANCE_VALUE) {
inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyTypeObject *tp = Py_TYPE(owner);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
uint32_t type_version = read_u32(cache->version);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
STAT_INC(STORE_ATTR, hit);
Py_ssize_t index = cache->index;
STACK_SHRINK(1);
PyObject *value = POP();
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyObject *old_value = values->values[index];
values->values[index] = value;
@ -1980,16 +1937,11 @@ dummy_func(
Py_DECREF(old_value);
}
Py_DECREF(owner);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
}
// stack effect: (__0, __1 -- )
inst(STORE_ATTR_WITH_HINT) {
inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyTypeObject *tp = Py_TYPE(owner);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
uint32_t type_version = read_u32(cache->version);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
@ -1999,17 +1951,14 @@ dummy_func(
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(names, oparg);
uint16_t hint = cache->index;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
PyObject *value, *old_value;
PyObject *old_value;
uint64_t new_version;
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
STACK_SHRINK(1);
value = POP();
new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
@ -2018,8 +1967,6 @@ dummy_func(
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
STACK_SHRINK(1);
value = POP();
new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
@ -2032,36 +1979,32 @@ dummy_func(
/* PEP 509 */
dict->ma_version_tag = new_version;
Py_DECREF(owner);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
}
// stack effect: (__0, __1 -- )
inst(STORE_ATTR_SLOT) {
inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyTypeObject *tp = Py_TYPE(owner);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
uint32_t type_version = read_u32(cache->version);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
char *addr = (char *)owner + cache->index;
char *addr = (char *)owner + index;
STAT_INC(STORE_ATTR, hit);
STACK_SHRINK(1);
PyObject *value = POP();
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = value;
Py_XDECREF(old_value);
Py_DECREF(owner);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
}
// stack effect: (__0 -- )
inst(COMPARE_OP) {
family(compare_op) = {
COMPARE_OP,
_COMPARE_OP_FLOAT,
_COMPARE_OP_INT,
_COMPARE_OP_STR,
};
inst(COMPARE_OP, (unused/2, left, right -- res)) {
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *right = TOP();
PyObject *left = SECOND();
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
@ -2069,57 +2012,43 @@ dummy_func(
STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
assert(oparg <= Py_GE);
PyObject *right = POP();
PyObject *left = TOP();
PyObject *res = PyObject_RichCompare(left, right, oparg);
SET_TOP(res);
res = PyObject_RichCompare(left, right, oparg);
Py_DECREF(left);
Py_DECREF(right);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
ERROR_IF(res == NULL, error);
}
// stack effect: (__0 -- )
inst(COMPARE_OP_FLOAT_JUMP) {
// The result is an int disguised as an object pointer.
op(_COMPARE_OP_FLOAT, (unused/1, when_to_jump_mask/1, left, right -- jump: size_t)) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
double dleft = PyFloat_AS_DOUBLE(left);
double dright = PyFloat_AS_DOUBLE(right);
int sign = (dleft > dright) - (dleft < dright);
// 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask
int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright);
DEOPT_IF(isnan(dleft), COMPARE_OP);
DEOPT_IF(isnan(dright), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
jump = sign_ish & when_to_jump_mask;
}
// The input is an int disguised as an object pointer!
op(_JUMP_IF, (jump: size_t --)) {
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
}
else {
JUMPBY(1 + oparg);
if (jump) {
JUMPBY(oparg);
}
}
// We're praying that the compiler optimizes the flags manipuations.
super(COMPARE_OP_FLOAT_JUMP) = _COMPARE_OP_FLOAT + _JUMP_IF;
// stack effect: (__0 -- )
inst(COMPARE_OP_INT_JUMP) {
// Similar to COMPARE_OP_FLOAT
op(_COMPARE_OP_INT, (unused/1, when_to_jump_mask/1, left, right -- jump: size_t)) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
@ -2128,51 +2057,30 @@ dummy_func(
assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
int sign = (ileft > iright) - (ileft < iright);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
// 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask
int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
}
else {
JUMPBY(1 + oparg);
}
jump = sign_ish & when_to_jump_mask;
}
super(COMPARE_OP_INT_JUMP) = _COMPARE_OP_INT + _JUMP_IF;
// stack effect: (__0 -- )
inst(COMPARE_OP_STR_JUMP) {
// Similar to COMPARE_OP_FLOAT, but for ==, != only
op(_COMPARE_OP_STR, (unused/1, invert/1, left, right -- jump: size_t)) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int invert = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
int res = _PyUnicode_Equal(left, right);
assert(oparg == Py_EQ || oparg == Py_NE);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(res == 0 || res == 1);
assert(invert == 0 || invert == 1);
int jump = res ^ invert;
if (!jump) {
next_instr++;
}
else {
JUMPBY(1 + oparg);
}
jump = res ^ invert;
}
super(COMPARE_OP_STR_JUMP) = _COMPARE_OP_STR + _JUMP_IF;
// stack effect: (__0 -- )
inst(IS_OP) {
@ -3633,7 +3541,7 @@ dummy_func(
PUSH(Py_NewRef(peek));
}
inst(BINARY_OP, (lhs, rhs, unused/1 -- res)) {
inst(BINARY_OP, (unused/1, lhs, rhs -- res)) {
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
@ -3691,9 +3599,6 @@ dummy_func(
// Future families go below this point //
family(binary_subscr) = {
BINARY_SUBSCR, BINARY_SUBSCR_DICT,
BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, BINARY_SUBSCR_TUPLE_INT };
family(call) = {
CALL, CALL_PY_EXACT_ARGS,
CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS,
@ -3702,9 +3607,6 @@ family(call) = {
CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1,
CALL_NO_KW_TYPE_1 };
family(compare_op) = {
COMPARE_OP, COMPARE_OP_FLOAT_JUMP,
COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP };
family(for_iter) = {
FOR_ITER, FOR_ITER_LIST,
FOR_ITER_RANGE };
@ -3719,13 +3621,7 @@ family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST };
family(load_global) = {
LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN,
LOAD_GLOBAL_MODULE };
family(store_attr) = {
STORE_ATTR, STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT };
family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST };
family(store_subscr) = {
STORE_SUBSCR, STORE_SUBSCR_DICT,
STORE_SUBSCR_LIST_INT };
family(unpack_sequence) = {
UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST,
UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE };

489
Python/generated_cases.c.h generated
View file

@ -139,7 +139,7 @@
if (prod == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, prod);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -159,7 +159,7 @@
if (prod == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, prod);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -177,7 +177,7 @@
if (sub == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, sub);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -196,7 +196,7 @@
if (sub == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, sub);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -214,7 +214,7 @@
if (res == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, res);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -268,7 +268,7 @@
if (sum == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, sum);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -286,7 +286,7 @@
if (sum == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, sum);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -311,7 +311,7 @@
if (res == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, res);
next_instr += 4;
JUMPBY(4);
DISPATCH();
}
@ -380,7 +380,7 @@
Py_DECREF(list);
STACK_SHRINK(1);
POKE(1, res);
next_instr += 4;
JUMPBY(4);
DISPATCH();
}
@ -406,7 +406,7 @@
Py_DECREF(tuple);
STACK_SHRINK(1);
POKE(1, res);
next_instr += 4;
JUMPBY(4);
DISPATCH();
}
@ -424,14 +424,14 @@
}
Py_DECREF(dict);
Py_DECREF(sub);
if (1) goto pop_2_error;
if (true) goto pop_2_error;
}
Py_INCREF(res); // Do this before DECREF'ing dict, sub
Py_DECREF(dict);
Py_DECREF(sub);
STACK_SHRINK(1);
POKE(1, res);
next_instr += 4;
JUMPBY(4);
DISPATCH();
}
@ -464,22 +464,21 @@
}
TARGET(LIST_APPEND) {
PyObject *v = POP();
PyObject *list = PEEK(oparg);
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
goto error;
PyObject *v = PEEK(1);
PyObject *list = PEEK(oparg + 1); // +1 to account for v staying on stack
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
STACK_SHRINK(1);
PREDICT(JUMP_BACKWARD);
DISPATCH();
}
TARGET(SET_ADD) {
PyObject *v = POP();
PyObject *set = PEEK(oparg);
int err;
err = PySet_Add(set, v);
PyObject *v = PEEK(1);
PyObject *set = PEEK(oparg + 1); // +1 to account for v staying on stack
int err = PySet_Add(set, v);
Py_DECREF(v);
if (err != 0)
goto error;
if (err) goto pop_1_error;
STACK_SHRINK(1);
PREDICT(JUMP_BACKWARD);
DISPATCH();
}
@ -489,31 +488,32 @@
PyObject *sub = PEEK(1);
PyObject *container = PEEK(2);
PyObject *v = PEEK(3);
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
uint16_t counter = read_u16(next_instr + 0);
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_SUBSCR, deferred);
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
/* container[sub] = v */
int err = PyObject_SetItem(container, sub, v);
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
if (err != 0) goto pop_3_error;
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
if (err) goto pop_3_error;
STACK_SHRINK(3);
JUMPBY(1);
DISPATCH();
}
TARGET(STORE_SUBSCR_LIST_INT) {
PyObject *sub = PEEK(1);
PyObject *list = PEEK(2);
PyObject *value = PEEK(3);
assert(cframe.use_tracing == 0);
PyObject *sub = TOP();
PyObject *list = SECOND();
PyObject *value = THIRD();
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
@ -526,61 +526,58 @@
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, value);
STACK_SHRINK(3);
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
STACK_SHRINK(3);
JUMPBY(1);
DISPATCH();
}
TARGET(STORE_SUBSCR_DICT) {
PyObject *sub = PEEK(1);
PyObject *dict = PEEK(2);
PyObject *value = PEEK(3);
assert(cframe.use_tracing == 0);
PyObject *sub = TOP();
PyObject *dict = SECOND();
PyObject *value = THIRD();
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
STACK_SHRINK(3);
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
if (err != 0) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
if (err) goto pop_3_error;
STACK_SHRINK(3);
JUMPBY(1);
DISPATCH();
}
TARGET(DELETE_SUBSCR) {
PyObject *sub = TOP();
PyObject *container = SECOND();
int err;
STACK_SHRINK(2);
PyObject *sub = PEEK(1);
PyObject *container = PEEK(2);
/* del container[sub] */
err = PyObject_DelItem(container, sub);
int err = PyObject_DelItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
if (err != 0)
goto error;
if (err) goto pop_2_error;
STACK_SHRINK(2);
DISPATCH();
}
TARGET(PRINT_EXPR) {
PyObject *value = POP();
PyObject *value = PEEK(1);
PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook));
PyObject *res;
// Can't use ERROR_IF here.
if (hook == NULL) {
_PyErr_SetString(tstate, PyExc_RuntimeError,
"lost sys.displayhook");
Py_DECREF(value);
goto error;
if (true) goto pop_1_error;
}
res = PyObject_CallOneArg(hook, value);
Py_DECREF(value);
if (res == NULL)
goto error;
if (res == NULL) goto pop_1_error;
Py_DECREF(res);
STACK_SHRINK(1);
DISPATCH();
}
@ -607,9 +604,10 @@
}
TARGET(INTERPRETER_EXIT) {
PyObject *retval = PEEK(1);
assert(frame == &entry_frame);
assert(_PyFrame_IsIncomplete(frame));
PyObject *retval = POP();
STACK_SHRINK(1); // Since we're not going to DISPATCH()
assert(EMPTY());
/* Restore previous cframe and return. */
tstate->cframe = cframe.previous;
@ -621,7 +619,8 @@
}
TARGET(RETURN_VALUE) {
PyObject *retval = POP();
PyObject *retval = PEEK(1);
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
TRACE_FUNCTION_EXIT();
@ -637,46 +636,39 @@
}
TARGET(GET_AITER) {
PyObject *obj = PEEK(1);
PyObject *iter;
unaryfunc getter = NULL;
PyObject *iter = NULL;
PyObject *obj = TOP();
PyTypeObject *type = Py_TYPE(obj);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
if (getter != NULL) {
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) {
SET_TOP(NULL);
goto error;
}
}
else {
SET_TOP(NULL);
if (getter == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
goto error;
if (true) goto pop_1_error;
}
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) goto pop_1_error;
if (Py_TYPE(iter)->tp_as_async == NULL ||
Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
SET_TOP(NULL);
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
goto error;
if (true) goto pop_1_error;
}
SET_TOP(iter);
POKE(1, iter);
DISPATCH();
}
@ -1131,51 +1123,46 @@
TARGET(STORE_ATTR) {
PREDICTED(STORE_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
PyObject *owner = PEEK(1);
PyObject *v = PEEK(2);
uint16_t counter = read_u16(next_instr + 0);
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg);
next_instr--;
_Py_Specialize_StoreAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_ATTR, deferred);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
PyObject *name = GETITEM(names, oparg);
PyObject *owner = TOP();
PyObject *v = SECOND();
int err;
STACK_SHRINK(2);
err = PyObject_SetAttr(owner, name, v);
int err = PyObject_SetAttr(owner, name, v);
Py_DECREF(v);
Py_DECREF(owner);
if (err != 0) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
if (err) goto pop_2_error;
STACK_SHRINK(2);
JUMPBY(4);
DISPATCH();
}
TARGET(DELETE_ATTR) {
PyObject *owner = PEEK(1);
PyObject *name = GETITEM(names, oparg);
PyObject *owner = POP();
int err;
err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
int err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
Py_DECREF(owner);
if (err != 0)
goto error;
if (err) goto pop_1_error;
STACK_SHRINK(1);
DISPATCH();
}
TARGET(STORE_GLOBAL) {
PyObject *v = PEEK(1);
PyObject *name = GETITEM(names, oparg);
PyObject *v = POP();
int err;
err = PyDict_SetItem(GLOBALS(), name, v);
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
if (err != 0)
goto error;
if (err) goto pop_1_error;
STACK_SHRINK(1);
DISPATCH();
}
@ -1970,20 +1957,18 @@
}
TARGET(STORE_ATTR_INSTANCE_VALUE) {
PyObject *owner = PEEK(1);
PyObject *value = PEEK(2);
uint32_t type_version = read_u32(next_instr + 1);
uint16_t index = read_u16(next_instr + 3);
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyTypeObject *tp = Py_TYPE(owner);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
uint32_t type_version = read_u32(cache->version);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
STAT_INC(STORE_ATTR, hit);
Py_ssize_t index = cache->index;
STACK_SHRINK(1);
PyObject *value = POP();
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyObject *old_value = values->values[index];
values->values[index] = value;
@ -1994,16 +1979,18 @@
Py_DECREF(old_value);
}
Py_DECREF(owner);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
STACK_SHRINK(2);
JUMPBY(4);
DISPATCH();
}
TARGET(STORE_ATTR_WITH_HINT) {
PyObject *owner = PEEK(1);
PyObject *value = PEEK(2);
uint32_t type_version = read_u32(next_instr + 1);
uint16_t hint = read_u16(next_instr + 3);
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyTypeObject *tp = Py_TYPE(owner);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
uint32_t type_version = read_u32(cache->version);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
@ -2013,17 +2000,14 @@
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(names, oparg);
uint16_t hint = cache->index;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
PyObject *value, *old_value;
PyObject *old_value;
uint64_t new_version;
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
STACK_SHRINK(1);
value = POP();
new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
@ -2032,8 +2016,6 @@
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
STACK_SHRINK(1);
value = POP();
new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
@ -2046,37 +2028,39 @@
/* PEP 509 */
dict->ma_version_tag = new_version;
Py_DECREF(owner);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
STACK_SHRINK(2);
JUMPBY(4);
DISPATCH();
}
TARGET(STORE_ATTR_SLOT) {
PyObject *owner = PEEK(1);
PyObject *value = PEEK(2);
uint32_t type_version = read_u32(next_instr + 1);
uint16_t index = read_u16(next_instr + 3);
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyTypeObject *tp = Py_TYPE(owner);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
uint32_t type_version = read_u32(cache->version);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
char *addr = (char *)owner + cache->index;
char *addr = (char *)owner + index;
STAT_INC(STORE_ATTR, hit);
STACK_SHRINK(1);
PyObject *value = POP();
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = value;
Py_XDECREF(old_value);
Py_DECREF(owner);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR);
STACK_SHRINK(2);
JUMPBY(4);
DISPATCH();
}
TARGET(COMPARE_OP) {
PREDICTED(COMPARE_OP);
PyObject *right = PEEK(1);
PyObject *left = PEEK(2);
PyObject *res;
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *right = TOP();
PyObject *left = SECOND();
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
@ -2084,109 +2068,13 @@
STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
assert(oparg <= Py_GE);
PyObject *right = POP();
PyObject *left = TOP();
PyObject *res = PyObject_RichCompare(left, right, oparg);
SET_TOP(res);
res = PyObject_RichCompare(left, right, oparg);
Py_DECREF(left);
Py_DECREF(right);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
DISPATCH();
}
TARGET(COMPARE_OP_FLOAT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
double dleft = PyFloat_AS_DOUBLE(left);
double dright = PyFloat_AS_DOUBLE(right);
int sign = (dleft > dright) - (dleft < dright);
DEOPT_IF(isnan(dleft), COMPARE_OP);
DEOPT_IF(isnan(dright), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
}
else {
JUMPBY(1 + oparg);
}
DISPATCH();
}
TARGET(COMPARE_OP_INT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
int sign = (ileft > iright) - (ileft < iright);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
}
else {
JUMPBY(1 + oparg);
}
DISPATCH();
}
TARGET(COMPARE_OP_STR_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int invert = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
int res = _PyUnicode_Equal(left, right);
assert(oparg == Py_EQ || oparg == Py_NE);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(res == 0 || res == 1);
assert(invert == 0 || invert == 1);
int jump = res ^ invert;
if (!jump) {
next_instr++;
}
else {
JUMPBY(1 + oparg);
}
if (res == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(2);
DISPATCH();
}
@ -3681,7 +3569,7 @@
if (res == NULL) goto pop_2_error;
STACK_SHRINK(1);
POKE(1, res);
next_instr += 1;
JUMPBY(1);
DISPATCH();
}
@ -3714,20 +3602,20 @@
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
_tmp_1 = value;
_tmp_2 = value;
}
NEXTOPARG();
next_instr++;
JUMPBY(1);
{
PyObject *value;
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
_tmp_2 = value;
_tmp_1 = value;
}
STACK_GROW(2);
POKE(1, _tmp_2);
POKE(2, _tmp_1);
POKE(1, _tmp_1);
POKE(2, _tmp_2);
DISPATCH();
}
@ -3739,19 +3627,19 @@
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
_tmp_1 = value;
_tmp_2 = value;
}
NEXTOPARG();
next_instr++;
JUMPBY(1);
{
PyObject *value;
value = GETITEM(consts, oparg);
Py_INCREF(value);
_tmp_2 = value;
_tmp_1 = value;
}
STACK_GROW(2);
POKE(1, _tmp_2);
POKE(2, _tmp_1);
POKE(1, _tmp_1);
POKE(2, _tmp_2);
DISPATCH();
}
@ -3762,7 +3650,7 @@
SETLOCAL(oparg, value);
}
NEXTOPARG();
next_instr++;
JUMPBY(1);
{
PyObject *value;
value = GETLOCAL(oparg);
@ -3775,16 +3663,16 @@
}
TARGET(STORE_FAST__STORE_FAST) {
PyObject *_tmp_1 = PEEK(2);
PyObject *_tmp_2 = PEEK(1);
PyObject *_tmp_1 = PEEK(1);
PyObject *_tmp_2 = PEEK(2);
{
PyObject *value = _tmp_2;
PyObject *value = _tmp_1;
SETLOCAL(oparg, value);
}
NEXTOPARG();
next_instr++;
JUMPBY(1);
{
PyObject *value = _tmp_1;
PyObject *value = _tmp_2;
SETLOCAL(oparg, value);
}
STACK_SHRINK(2);
@ -3798,32 +3686,145 @@
PyObject *value;
value = GETITEM(consts, oparg);
Py_INCREF(value);
_tmp_1 = value;
_tmp_2 = value;
}
NEXTOPARG();
next_instr++;
JUMPBY(1);
{
PyObject *value;
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
_tmp_2 = value;
_tmp_1 = value;
}
STACK_GROW(2);
POKE(1, _tmp_2);
POKE(2, _tmp_1);
POKE(1, _tmp_1);
POKE(2, _tmp_2);
DISPATCH();
}
TARGET(COMPARE_OP_FLOAT_JUMP) {
PyObject *_tmp_1 = PEEK(1);
PyObject *_tmp_2 = PEEK(2);
{
PyObject *right = _tmp_1;
PyObject *left = _tmp_2;
size_t jump;
uint16_t when_to_jump_mask = read_u16(next_instr + 1);
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
double dleft = PyFloat_AS_DOUBLE(left);
double dright = PyFloat_AS_DOUBLE(right);
// 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask
int sign_ish = 2*(dleft > dright) + 2 - (dleft < dright);
DEOPT_IF(isnan(dleft), COMPARE_OP);
DEOPT_IF(isnan(dright), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
jump = sign_ish & when_to_jump_mask;
_tmp_2 = (PyObject *)jump;
}
JUMPBY(2);
NEXTOPARG();
JUMPBY(1);
{
size_t jump = (size_t)_tmp_2;
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
if (jump) {
JUMPBY(oparg);
}
}
STACK_SHRINK(2);
DISPATCH();
}
TARGET(COMPARE_OP_INT_JUMP) {
PyObject *_tmp_1 = PEEK(1);
PyObject *_tmp_2 = PEEK(2);
{
PyObject *right = _tmp_1;
PyObject *left = _tmp_2;
size_t jump;
uint16_t when_to_jump_mask = read_u16(next_instr + 1);
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
// 1 if <, 2 if ==, 4 if >; this matches when _to_jump_mask
int sign_ish = 2*(ileft > iright) + 2 - (ileft < iright);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
jump = sign_ish & when_to_jump_mask;
_tmp_2 = (PyObject *)jump;
}
JUMPBY(2);
NEXTOPARG();
JUMPBY(1);
{
size_t jump = (size_t)_tmp_2;
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
if (jump) {
JUMPBY(oparg);
}
}
STACK_SHRINK(2);
DISPATCH();
}
TARGET(COMPARE_OP_STR_JUMP) {
PyObject *_tmp_1 = PEEK(1);
PyObject *_tmp_2 = PEEK(2);
{
PyObject *right = _tmp_1;
PyObject *left = _tmp_2;
size_t jump;
uint16_t invert = read_u16(next_instr + 1);
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
int res = _PyUnicode_Equal(left, right);
assert(oparg == Py_EQ || oparg == Py_NE);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(res == 0 || res == 1);
assert(invert == 0 || invert == 1);
jump = res ^ invert;
_tmp_2 = (PyObject *)jump;
}
JUMPBY(2);
NEXTOPARG();
JUMPBY(1);
{
size_t jump = (size_t)_tmp_2;
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
if (jump) {
JUMPBY(oparg);
}
}
STACK_SHRINK(2);
DISPATCH();
}
TARGET(END_FOR) {
PyObject *_tmp_1 = PEEK(2);
PyObject *_tmp_2 = PEEK(1);
PyObject *_tmp_1 = PEEK(1);
PyObject *_tmp_2 = PEEK(2);
{
PyObject *value = _tmp_2;
PyObject *value = _tmp_1;
Py_DECREF(value);
}
{
PyObject *value = _tmp_1;
PyObject *value = _tmp_2;
Py_DECREF(value);
}
STACK_SHRINK(2);