Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-01-03 14:02:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-140683: JIT: Improve machine code for loading smaller constants on AArch64. (GH-142511)

Browse source 
* Use movz and movk instructions for loading 16 and 32 bit operands and oparg.
* Loading of 64 bit operands is unchanged.
This commit is contained in:
Mark Shannon 2025-12-11 12:33:39 +00:00 • committed by GitHub
parent 469f191a85
commit 4eab90f4f3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 320 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Python/bytecodes.c
View file

@ -810,7 +810,7 @@ dummy_func(
assert(next_instr->op.code == STORE_FAST);
next_oparg = next_instr->op.arg;
#else
next_oparg = (int)CURRENT_OPERAND0();
next_oparg = (int)CURRENT_OPERAND0_16();
#endif
_PyStackRef *target_local = &GETLOCAL(next_oparg);
assert(PyUnicode_CheckExact(left_o));

8
Python/ceval_macros.h
View file

@ -450,8 +450,12 @@ do { \
} while (0)
#define CURRENT_OPARG() (next_uop[-1].oparg)
#define CURRENT_OPERAND0() (next_uop[-1].operand0)
#define CURRENT_OPERAND1() (next_uop[-1].operand1)
#define CURRENT_OPERAND0_64() (next_uop[-1].operand0)
#define CURRENT_OPERAND1_64() (next_uop[-1].operand1)
#define CURRENT_OPERAND0_32() (next_uop[-1].operand0)
#define CURRENT_OPERAND1_32() (next_uop[-1].operand1)
#define CURRENT_OPERAND0_16() (next_uop[-1].operand0)
#define CURRENT_OPERAND1_16() (next_uop[-1].operand1)
#define CURRENT_TARGET() (next_uop[-1].target)
#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target

234
Python/executor_cases.c.h generated
View file

@ -4645,7 +4645,7 @@
assert(next_instr->op.code == STORE_FAST);
next_oparg = next_instr->op.arg;
#else
next_oparg = (int)CURRENT_OPERAND0();
next_oparg = (int)CURRENT_OPERAND0_16();
#endif
_PyStackRef *target_local = &GETLOCAL(next_oparg);
assert(PyUnicode_CheckExact(left_o));
@ -4692,7 +4692,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
_PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr;
@ -4730,7 +4730,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5);
@ -6736,7 +6736,7 @@
case _GUARD_GLOBALS_VERSION_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
uint16_t version = (uint16_t)CURRENT_OPERAND0();
uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@ -6759,7 +6759,7 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
uint16_t version = (uint16_t)CURRENT_OPERAND0();
uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@ -6784,7 +6784,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
uint16_t version = (uint16_t)CURRENT_OPERAND0();
uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@ -6811,7 +6811,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
uint16_t version = (uint16_t)CURRENT_OPERAND0();
uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@ -6837,8 +6837,8 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef res;
uint16_t version = (uint16_t)CURRENT_OPERAND0();
uint16_t index = (uint16_t)CURRENT_OPERAND1();
uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
uint16_t index = (uint16_t)CURRENT_OPERAND1_16();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@ -6883,8 +6883,8 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef res;
uint16_t version = (uint16_t)CURRENT_OPERAND0();
uint16_t index = (uint16_t)CURRENT_OPERAND1();
uint16_t version = (uint16_t)CURRENT_OPERAND0_16();
uint16_t index = (uint16_t)CURRENT_OPERAND1_16();
PyDictObject *dict = (PyDictObject *)BUILTINS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
@ -7936,7 +7936,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@ -7958,7 +7958,7 @@
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@ -7979,7 +7979,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@ -8002,7 +8002,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) {
@ -8023,7 +8023,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
if (!LOCK_OBJECT(owner_o)) {
@ -8054,7 +8054,7 @@
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
if (!LOCK_OBJECT(owner_o)) {
@ -8084,7 +8084,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
if (!LOCK_OBJECT(owner_o)) {
@ -8116,7 +8116,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(type_version != 0);
if (!LOCK_OBJECT(owner_o)) {
@ -8235,7 +8235,7 @@
_PyStackRef attr;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
uint16_t offset = (uint16_t)CURRENT_OPERAND0_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
@ -8280,8 +8280,8 @@
_PyStackRef attr;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint32_t dict_version = (uint32_t)CURRENT_OPERAND0();
uint16_t index = (uint16_t)CURRENT_OPERAND1();
uint32_t dict_version = (uint32_t)CURRENT_OPERAND0_32();
uint16_t index = (uint16_t)CURRENT_OPERAND1_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
UOP_STAT_INC(uopcode, miss);
@ -8342,7 +8342,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
uint16_t hint = (uint16_t)CURRENT_OPERAND0();
uint16_t hint = (uint16_t)CURRENT_OPERAND0_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
@ -8428,7 +8428,7 @@
_PyStackRef attr;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint16_t index = (uint16_t)CURRENT_OPERAND0();
uint16_t index = (uint16_t)CURRENT_OPERAND0_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **addr = (PyObject **)((char *)owner_o + index);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
@ -8472,7 +8472,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!PyType_Check(owner_o)) {
UOP_STAT_INC(uopcode, miss);
@ -8499,7 +8499,7 @@
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!PyType_Check(owner_o)) {
UOP_STAT_INC(uopcode, miss);
@ -8525,7 +8525,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!PyType_Check(owner_o)) {
UOP_STAT_INC(uopcode, miss);
@ -8553,7 +8553,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!PyType_Check(owner_o)) {
UOP_STAT_INC(uopcode, miss);
@ -8581,7 +8581,7 @@
_PyStackRef attr;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
@ -8612,7 +8612,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
PyObject *fget = (PyObject *)CURRENT_OPERAND0();
PyObject *fget = (PyObject *)CURRENT_OPERAND0_64();
assert((oparg & 1) == 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
@ -8763,7 +8763,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
value = _stack_item_0;
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
uint16_t offset = (uint16_t)CURRENT_OPERAND0_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
STAT_INC(STORE_ATTR, hit);
assert(_PyObject_GetManagedDict(owner_o) == NULL);
@ -8798,7 +8798,7 @@
oparg = CURRENT_OPARG();
owner = _stack_item_1;
value = _stack_item_0;
uint16_t hint = (uint16_t)CURRENT_OPERAND0();
uint16_t hint = (uint16_t)CURRENT_OPERAND0_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
@ -8874,7 +8874,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
value = _stack_item_0;
uint16_t index = (uint16_t)CURRENT_OPERAND0();
uint16_t index = (uint16_t)CURRENT_OPERAND0_16();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
if (!LOCK_OBJECT(owner_o)) {
UOP_STAT_INC(uopcode, miss);
@ -11133,7 +11133,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef owner;
owner = stack_pointer[-1];
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@ -11156,7 +11156,7 @@
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@ -11178,7 +11178,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@ -11202,7 +11202,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0();
uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32();
PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
PyDictKeysObject *keys = owner_heap_type->ht_cached_keys;
@ -11227,7 +11227,7 @@
_PyStackRef self;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11252,7 +11252,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11276,7 +11276,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
oparg = CURRENT_OPARG();
owner = _stack_item_1;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11299,7 +11299,7 @@
_PyStackRef self;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
@ -11325,7 +11325,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
@ -11350,7 +11350,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
oparg = CURRENT_OPARG();
owner = _stack_item_1;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
@ -11374,7 +11374,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11398,7 +11398,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert((oparg & 1) == 0);
assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
@ -11420,7 +11420,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef owner;
owner = stack_pointer[-1];
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
if (dict != NULL) {
@ -11442,7 +11442,7 @@
_PyStackRef owner;
_PyStackRef _stack_item_0 = _tos_cache0;
owner = _stack_item_0;
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
if (dict != NULL) {
@ -11463,7 +11463,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
owner = _stack_item_1;
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
if (dict != NULL) {
@ -11486,7 +11486,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
owner = _stack_item_2;
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0();
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16();
char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr);
if (dict != NULL) {
@ -11510,7 +11510,7 @@
_PyStackRef self;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11535,7 +11535,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
owner = _stack_item_0;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11559,7 +11559,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
oparg = CURRENT_OPARG();
owner = _stack_item_1;
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
PyObject *descr = (PyObject *)CURRENT_OPERAND0_64();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
@ -11656,7 +11656,7 @@
_PyStackRef callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (!PyFunction_Check(callable_o)) {
UOP_STAT_INC(uopcode, miss);
@ -11677,8 +11677,8 @@
case _CHECK_FUNCTION_VERSION_INLINE_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
@ -11695,8 +11695,8 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
@ -11715,8 +11715,8 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
@ -11737,8 +11737,8 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64();
assert(PyFunction_Check(callable_o));
PyFunctionObject *func = (PyFunctionObject *)callable_o;
if (func->func_version != func_version) {
@ -11762,7 +11762,7 @@
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (Py_TYPE(callable_o) != &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
@ -12941,7 +12941,7 @@
oparg = CURRENT_OPARG();
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (!PyStackRef_IsNull(self_or_null)) {
UOP_STAT_INC(uopcode, miss);
@ -14123,7 +14123,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (!PyFunction_Check(callable_o)) {
UOP_STAT_INC(uopcode, miss);
@ -14151,7 +14151,7 @@
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32();
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
if (Py_TYPE(callable_o) != &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
@ -15403,7 +15403,7 @@
case _SET_IP_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
SET_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
@ -15414,7 +15414,7 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
@ -15427,7 +15427,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
@ -15442,7 +15442,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64();
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
@ -15455,7 +15455,7 @@
case _CHECK_STACK_SPACE_OPERAND_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
assert(framesize <= INT_MAX);
if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
UOP_STAT_INC(uopcode, miss);
@ -15476,7 +15476,7 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
assert(framesize <= INT_MAX);
if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
UOP_STAT_INC(uopcode, miss);
@ -15499,7 +15499,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
assert(framesize <= INT_MAX);
if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
UOP_STAT_INC(uopcode, miss);
@ -15524,7 +15524,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
uint32_t framesize = (uint32_t)CURRENT_OPERAND0();
uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32();
assert(framesize <= INT_MAX);
if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
UOP_STAT_INC(uopcode, miss);
@ -15619,7 +15619,7 @@
case _EXIT_TRACE_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
_PyExitData *exit = (_PyExitData *)exit_p;
#if defined(Py_DEBUG) && !defined(_Py_JIT)
const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@ -15646,7 +15646,7 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
_PyExitData *exit = (_PyExitData *)exit_p;
#if defined(Py_DEBUG) && !defined(_Py_JIT)
const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@ -15681,7 +15681,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
_PyExitData *exit = (_PyExitData *)exit_p;
#if defined(Py_DEBUG) && !defined(_Py_JIT)
const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@ -15719,7 +15719,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
_PyExitData *exit = (_PyExitData *)exit_p;
#if defined(Py_DEBUG) && !defined(_Py_JIT)
const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER)
@ -15756,7 +15756,7 @@
case _DYNAMIC_EXIT_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
#if defined(Py_DEBUG) && !defined(_Py_JIT)
_PyExitData *exit = (_PyExitData *)exit_p;
_Py_CODEUNIT *target = frame->instr_ptr;
@ -15780,7 +15780,7 @@
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
#if defined(Py_DEBUG) && !defined(_Py_JIT)
_PyExitData *exit = (_PyExitData *)exit_p;
_Py_CODEUNIT *target = frame->instr_ptr;
@ -15812,7 +15812,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
#if defined(Py_DEBUG) && !defined(_Py_JIT)
_PyExitData *exit = (_PyExitData *)exit_p;
_Py_CODEUNIT *target = frame->instr_ptr;
@ -15847,7 +15847,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0();
PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64();
#if defined(Py_DEBUG) && !defined(_Py_JIT)
_PyExitData *exit = (_PyExitData *)exit_p;
_Py_CODEUNIT *target = frame->instr_ptr;
@ -15946,7 +15946,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
value = PyStackRef_FromPyObjectNew(ptr);
_tos_cache0 = value;
SET_CURRENT_CACHED_VALUES(1);
@ -15959,7 +15959,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef value;
_PyStackRef _stack_item_0 = _tos_cache0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
value = PyStackRef_FromPyObjectNew(ptr);
_tos_cache1 = value;
_tos_cache0 = _stack_item_0;
@ -15974,7 +15974,7 @@
_PyStackRef value;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
value = PyStackRef_FromPyObjectNew(ptr);
_tos_cache2 = value;
_tos_cache1 = _stack_item_1;
@ -15991,7 +15991,7 @@
_PyStackRef value;
_PyStackRef _stack_item_0 = _tos_cache0;
pop = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop);
stack_pointer = _PyFrame_GetStackPointer(frame);
@ -16008,7 +16008,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
value = PyStackRef_FromPyObjectBorrow(ptr);
_tos_cache0 = value;
SET_CURRENT_CACHED_VALUES(1);
@ -16021,7 +16021,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef value;
_PyStackRef _stack_item_0 = _tos_cache0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
value = PyStackRef_FromPyObjectBorrow(ptr);
_tos_cache1 = value;
_tos_cache0 = _stack_item_0;
@ -16036,7 +16036,7 @@
_PyStackRef value;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
value = PyStackRef_FromPyObjectBorrow(ptr);
_tos_cache2 = value;
_tos_cache1 = _stack_item_1;
@ -16147,7 +16147,7 @@
_PyStackRef value;
_PyStackRef _stack_item_0 = _tos_cache0;
pop = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(pop);
stack_pointer = _PyFrame_GetStackPointer(frame);
@ -16170,7 +16170,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
pop2 = _stack_item_1;
pop1 = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
stack_pointer[0] = pop1;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@ -16201,7 +16201,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
null = _stack_item_1;
callable = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
(void)null;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(callable);
@ -16228,7 +16228,7 @@
pop = _stack_item_2;
null = _stack_item_1;
callable = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
stack_pointer[0] = callable;
stack_pointer[1] = null;
stack_pointer += 2;
@ -16266,7 +16266,7 @@
pop1 = _stack_item_1;
null = _stack_item_0;
callable = stack_pointer[-1];
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
stack_pointer[0] = null;
stack_pointer[1] = pop1;
stack_pointer += 2;
@ -16301,7 +16301,7 @@
_PyStackRef value;
_PyStackRef new;
old = stack_pointer[-1];
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
new = old;
value = PyStackRef_FromPyObjectNew(ptr);
_tos_cache1 = new;
@ -16321,7 +16321,7 @@
_PyStackRef new;
_PyStackRef _stack_item_0 = _tos_cache0;
old = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
new = old;
value = PyStackRef_FromPyObjectNew(ptr);
_tos_cache1 = new;
@ -16340,7 +16340,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
old = _stack_item_1;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
new = old;
value = PyStackRef_FromPyObjectNew(ptr);
_tos_cache2 = new;
@ -16358,7 +16358,7 @@
_PyStackRef value;
_PyStackRef new;
old = stack_pointer[-1];
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
new = old;
value = PyStackRef_FromPyObjectBorrow(ptr);
_tos_cache1 = new;
@ -16378,7 +16378,7 @@
_PyStackRef new;
_PyStackRef _stack_item_0 = _tos_cache0;
old = _stack_item_0;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
new = old;
value = PyStackRef_FromPyObjectBorrow(ptr);
_tos_cache1 = new;
@ -16397,7 +16397,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
old = _stack_item_1;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64();
new = old;
value = PyStackRef_FromPyObjectBorrow(ptr);
_tos_cache2 = new;
@ -16411,7 +16411,7 @@
case _START_EXECUTOR_r00: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
PyObject *executor = (PyObject *)CURRENT_OPERAND0();
PyObject *executor = (PyObject *)CURRENT_OPERAND0_64();
#ifndef _Py_JIT
assert(current_executor == (_PyExecutorObject*)executor);
#endif
@ -16646,7 +16646,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
oparg = CURRENT_OPARG();
uint32_t target = (uint32_t)CURRENT_OPERAND0();
uint32_t target = (uint32_t)CURRENT_OPERAND0_32();
assert(oparg == 0);
frame->instr_ptr = _PyFrame_GetBytecode(frame) + target;
SET_CURRENT_CACHED_VALUES(0);
@ -16972,7 +16972,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@ -16993,7 +16993,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@ -17016,7 +17016,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@ -17041,7 +17041,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF__PUSH_FRAME ((0))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF__PUSH_FRAME;
@ -17064,7 +17064,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@ -17085,7 +17085,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@ -17108,7 +17108,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@ -17133,7 +17133,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_YIELD_VALUE;
@ -17156,7 +17156,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@ -17177,7 +17177,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@ -17200,7 +17200,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@ -17225,7 +17225,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF_RETURN_VALUE ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_VALUE;
@ -17248,7 +17248,7 @@
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
@ -17269,7 +17269,7 @@
assert(WITHIN_STACK_BOUNDS_WITH_CACHE());
_PyStackRef _stack_item_0 = _tos_cache0;
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
@ -17292,7 +17292,7 @@
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;
@ -17317,7 +17317,7 @@
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
#define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset))
PyObject *ip = (PyObject *)CURRENT_OPERAND0();
PyObject *ip = (PyObject *)CURRENT_OPERAND0_64();
_Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR;
if (target != (_Py_CODEUNIT *)ip) {
frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR;

2
Python/generated_cases.c.h generated
View file

@ -371,7 +371,7 @@
assert(next_instr->op.code == STORE_FAST);
next_oparg = next_instr->op.arg;
#else
next_oparg = (int)CURRENT_OPERAND0();
next_oparg = (int)CURRENT_OPERAND0_16();
#endif
_PyStackRef *target_local = &GETLOCAL(next_oparg);
assert(PyUnicode_CheckExact(left_o));

7
Tools/cases_generator/tier2_generator.py
View file

@ -222,12 +222,13 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, t
idx = 0
for cache in uop.caches:
if cache.name != "unused":
bits = cache.size*16
if cache.size == 4:
type = cast = "PyObject *"
else:
type = f"uint{cache.size*16}_t "
cast = f"uint{cache.size*16}_t"
emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n")
type = f"uint{bits}_t "
cast = f"uint{bits}_t"
emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}_{bits}();\n")
idx += 1
reachable, storage = emitter.emit_tokens(uop, storage, None, False)
if reachable:

129
Tools/jit/_optimizers.py
View file

@ -96,6 +96,8 @@ class InstructionKind(enum.Enum):
LONG_BRANCH = enum.auto()
SHORT_BRANCH = enum.auto()
RETURN = enum.auto()
SMALL_CONST_1 = enum.auto()
SMALL_CONST_2 = enum.auto()
OTHER = enum.auto()
@ -172,6 +174,7 @@ class Optimizer:
)
# Override everything that follows in subclasses:
_supports_external_relocations = True
supports_small_constants = False
_branches: typing.ClassVar[dict[str, tuple[str | None, str | None]]] = {}
# Short branches are instructions that can branch within a micro-op,
# but might not have the reach to branch anywhere within a trace.
@ -184,6 +187,9 @@ class Optimizer:
_re_return: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH
text: str = ""
globals: set[str] = dataclasses.field(default_factory=set)
_re_small_const_1 = _RE_NEVER_MATCH
_re_small_const_2 = _RE_NEVER_MATCH
const_reloc = "<Not supported>"
def __post_init__(self) -> None:
# Split the code into a linked list of basic blocks. A basic block is an
@ -253,6 +259,14 @@ def _parse_instruction(self, line: str) -> Instruction:
elif match := self._re_return.match(line):
name = line
kind = InstructionKind.RETURN
elif match := self._re_small_const_1.match(line):
target = match["value"]
name = match["instruction"]
kind = InstructionKind.SMALL_CONST_1
elif match := self._re_small_const_2.match(line):
target = match["value"]
name = match["instruction"]
kind = InstructionKind.SMALL_CONST_2
else:
name, *_ = line.split(" ")
kind = InstructionKind.OTHER
@ -385,7 +399,7 @@ def _remove_redundant_jumps(self) -> None:
block.fallthrough = True
block.instructions.pop()
# Before:
# br ? FOO:
# branch FOO:
# ...
# FOO:
# jump BAR
@ -461,6 +475,70 @@ def _fixup_external_labels(self) -> None:
)
block.instructions.append(branch.update_target("0"))
def _make_temp_label(self, index: int) -> Instruction:
marker = f"jit_temp_{index}:"
return Instruction(InstructionKind.OTHER, "", marker, None)
def _fixup_constants(self) -> None:
if not self.supports_small_constants:
return
index = 0
for block in self._blocks():
fixed: list[Instruction] = []
small_const_index = -1
for inst in block.instructions:
if inst.kind == InstructionKind.SMALL_CONST_1:
marker = f"jit_pending_{inst.target}{index}:"
fixed.append(self._make_temp_label(index))
index += 1
small_const_index = len(fixed)
fixed.append(inst)
elif inst.kind == InstructionKind.SMALL_CONST_2:
if small_const_index < 0:
fixed.append(inst)
continue
small_const_1 = fixed[small_const_index]
if not self._small_consts_match(small_const_1, inst):
small_const_index = -1
fixed.append(inst)
continue
assert small_const_1.target is not None
if small_const_1.target.endswith("16"):
fixed[small_const_index] = self._make_temp_label(index)
index += 1
else:
assert small_const_1.target.endswith("32")
patch_kind, replacement = self._small_const_1(small_const_1)
if replacement is not None:
label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{small_const_1.target[:-3]}_JIT_RELOCATION_{index}:"
index += 1
fixed[small_const_index - 1] = Instruction(
InstructionKind.OTHER, "", label, None
)
fixed[small_const_index] = replacement
patch_kind, replacement = self._small_const_2(inst)
if replacement is not None:
assert inst.target is not None
label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{inst.target[:-3]}_JIT_RELOCATION_{index}:"
index += 1
fixed.append(
Instruction(InstructionKind.OTHER, "", label, None)
)
fixed.append(replacement)
small_const_index = -1
else:
fixed.append(inst)
block.instructions = fixed
def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]:
raise NotImplementedError()
def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]:
raise NotImplementedError()
def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool:
raise NotImplementedError()
def run(self) -> None:
"""Run this optimizer."""
self._insert_continue_label()
@ -472,6 +550,7 @@ def run(self) -> None:
self._remove_redundant_jumps()
self._remove_unreachable()
self._fixup_external_labels()
self._fixup_constants()
self.path.write_text(self._body())
@ -492,6 +571,54 @@ class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods
# https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/RET--Return-from-subroutine-
_re_return = re.compile(r"\s*ret\b")
supports_small_constants = True
_re_small_const_1 = re.compile(
r"\s*(?P<instruction>adrp)\s+.*(?P<value>_JIT_OP(ARG|ERAND(0|1))_(16|32)).*"
)
_re_small_const_2 = re.compile(
r"\s*(?P<instruction>ldr)\s+.*(?P<value>_JIT_OP(ARG|ERAND(0|1))_(16|32)).*"
)
const_reloc = "CUSTOM_AARCH64_CONST"
def _get_reg(self, inst: Instruction) -> str:
_, rest = inst.text.split(inst.name)
reg, *_ = rest.split(",")
return reg.strip()
def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]:
assert inst.kind is InstructionKind.SMALL_CONST_1
assert inst.target is not None
if "16" in inst.target:
return "", None
pre, _ = inst.text.split(inst.name)
return "16a", Instruction(
InstructionKind.OTHER, "movz", f"{pre}movz {self._get_reg(inst)}, 0", None
)
def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]:
assert inst.kind is InstructionKind.SMALL_CONST_2
assert inst.target is not None
pre, _ = inst.text.split(inst.name)
if "16" in inst.target:
return "16a", Instruction(
InstructionKind.OTHER,
"movz",
f"{pre}movz {self._get_reg(inst)}, 0",
None,
)
else:
return "16b", Instruction(
InstructionKind.OTHER,
"movk",
f"{pre}movk {self._get_reg(inst)}, 0, lsl #16",
None,
)
def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool:
reg1 = self._get_reg(inst1)
reg2 = self._get_reg(inst2)
return reg1 == reg2
class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods
"""i686-pc-windows-msvc/x86_64-apple-darwin/x86_64-unknown-linux-gnu"""

3
Tools/jit/_schema.py
View file

@ -9,6 +9,9 @@
"ARM64_RELOC_PAGE21",
"ARM64_RELOC_PAGEOFF12",
"ARM64_RELOC_UNSIGNED",
"CUSTOM_AARCH64_BRANCH19",
"CUSTOM_AARCH64_CONST_16",
"CUSTOM_AARCH64_CONST_32",
"IMAGE_REL_AMD64_REL32",
"IMAGE_REL_ARM64_BRANCH19",
"IMAGE_REL_ARM64_BRANCH26",

19
Tools/jit/_stencils.py
View file

@ -32,6 +32,12 @@ class HoleValue(enum.Enum):
# The current uop's operand0 on 32-bit platforms (exposed as _JIT_OPERAND0_HI/LO):
OPERAND0_HI = enum.auto()
OPERAND0_LO = enum.auto()
# 16 and 32 bit versions of OPARG, OPERAND0 and OPERAND1
OPARG_16 = enum.auto()
OPERAND0_16 = enum.auto()
OPERAND1_16 = enum.auto()
OPERAND0_32 = enum.auto()
OPERAND1_32 = enum.auto()
# The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND1):
OPERAND1 = enum.auto()
# The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND1_HI/LO):
@ -59,6 +65,8 @@ class HoleValue(enum.Enum):
"ARM64_RELOC_PAGEOFF12": "patch_aarch64_12",
"ARM64_RELOC_UNSIGNED": "patch_64",
"CUSTOM_AARCH64_BRANCH19": "patch_aarch64_19r",
"CUSTOM_AARCH64_CONST16a": "patch_aarch64_16a",
"CUSTOM_AARCH64_CONST16b": "patch_aarch64_16b",
# x86_64-pc-windows-msvc:
"IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx",
# aarch64-pc-windows-msvc:
@ -95,6 +103,7 @@ class HoleValue(enum.Enum):
"X86_64_RELOC_SIGNED": "patch_32r",
"X86_64_RELOC_UNSIGNED": "patch_64",
}
# Translate HoleValues to C expressions:
_HOLE_EXPRS = {
HoleValue.CODE: "(uintptr_t)code",
@ -103,10 +112,15 @@ class HoleValue(enum.Enum):
HoleValue.GOT: "",
# These should all have been turned into DATA values by process_relocations:
HoleValue.OPARG: "instruction->oparg",
HoleValue.OPARG_16: "instruction->oparg",
HoleValue.OPERAND0: "instruction->operand0",
HoleValue.OPERAND0_16: "instruction->operand0",
HoleValue.OPERAND0_32: "instruction->operand0",
HoleValue.OPERAND0_HI: "(instruction->operand0 >> 32)",
HoleValue.OPERAND0_LO: "(instruction->operand0 & UINT32_MAX)",
HoleValue.OPERAND1: "instruction->operand1",
HoleValue.OPERAND1_16: "instruction->operand1",
HoleValue.OPERAND1_32: "instruction->operand1",
HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)",
HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)",
HoleValue.TARGET: "instruction->target",
@ -201,7 +215,10 @@ def as_c(self, where: str) -> str:
if self.symbol:
if value:
value += " + "
value += f"(uintptr_t)&{self.symbol}"
if self.symbol.startswith("CONST"):
value += f"instruction->{self.symbol[10:].lower()}"
else:
value += f"(uintptr_t)&{self.symbol}"
if _signed(self.addend) or not value:
if value:
value += " + "

1
Tools/jit/_targets.py
View file

@ -138,6 +138,7 @@ async def _compile(
f"--target={self.triple}",
"-DPy_BUILD_CORE_MODULE",
"-D_DEBUG" if self.debug else "-DNDEBUG",
f"-DSUPPORTS_SMALL_CONSTS={1 if self.optimizer.supports_small_constants else 0}",
f"-D_JIT_OPCODE={opname}",
"-D_PyJIT_ACTIVE",
"-D_Py_JIT",

50
Tools/jit/template.c
View file

@ -34,14 +34,38 @@
#include "jit.h"
#undef CURRENT_OPERAND0_64
#define CURRENT_OPERAND0_64() (_operand0_64)
#undef CURRENT_OPERAND1_64
#define CURRENT_OPERAND1_64() (_operand1_64)
#undef CURRENT_OPARG
#undef CURRENT_OPERAND0_16
#undef CURRENT_OPERAND0_32
#undef CURRENT_OPERAND1_16
#undef CURRENT_OPERAND1_32
#if SUPPORTS_SMALL_CONSTS
#define CURRENT_OPARG() (_oparg_16)
#define CURRENT_OPERAND0_32() (_operand0_32)
#define CURRENT_OPERAND0_16() (_operand0_16)
#define CURRENT_OPERAND1_32() (_operand1_32)
#define CURRENT_OPERAND1_16() (_operand1_16)
#else
#define CURRENT_OPARG() (_oparg)
#define CURRENT_OPERAND0_32() (_operand0_64)
#define CURRENT_OPERAND0_16() (_operand0_64)
#define CURRENT_OPERAND1_32() (_operand1_64)
#define CURRENT_OPERAND1_16() (_operand1_64)
#undef CURRENT_OPERAND0
#define CURRENT_OPERAND0() (_operand0)
#endif
#undef CURRENT_OPERAND1
#define CURRENT_OPERAND1() (_operand1)
#undef CURRENT_TARGET
#define CURRENT_TARGET() (_target)
@ -105,18 +129,26 @@ _JIT_ENTRY(
int uopcode = _JIT_OPCODE;
_Py_CODEUNIT *next_instr;
// Other stuff we need handy:
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
#if SIZEOF_VOID_P == 8
PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0)
PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1)
PATCH_VALUE(uint64_t, _operand0_64, _JIT_OPERAND0)
PATCH_VALUE(uint64_t, _operand1_64, _JIT_OPERAND1)
#else
assert(SIZEOF_VOID_P == 4);
PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI)
PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO)
uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo;
uint64_t _operand0_64 = ((uint64_t)_operand0_hi << 32) | _operand0_lo;
PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI)
PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO)
uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo;
uint64_t _operand1_64 = ((uint64_t)_operand1_hi << 32) | _operand1_lo;
#endif
#if SUPPORTS_SMALL_CONSTS
PATCH_VALUE(uint32_t, _operand0_32, _JIT_OPERAND0_32)
PATCH_VALUE(uint32_t, _operand1_32, _JIT_OPERAND1_32)
PATCH_VALUE(uint16_t, _operand0_16, _JIT_OPERAND0_16)
PATCH_VALUE(uint16_t, _operand1_16, _JIT_OPERAND1_16)
PATCH_VALUE(uint16_t, _oparg_16, _JIT_OPARG_16)
#else
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
#endif
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
OPT_STAT_INC(uops_executed);