Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-15 00:00:57 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-146640: Optimize int operations by mutating uniquely-referenced operands in place (JIT only) (GH-146641)

Browse source
This commit is contained in:
Pieter Eendebak 2026-04-03 17:23:04 +02:00 • committed by GitHub
parent 80ab6d958a
commit 48317feec8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 2331 additions and 1262 deletions
Download patch file Download diff file Expand all files Collapse all files

57
Python/bytecodes.c
View file

@ -708,6 +708,63 @@ dummy_func(
macro(BINARY_OP_SUBTRACT_INT) =
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT;
// Inplace compact int ops: mutate the uniquely-referenced operand
// when possible. The op handles decref of TARGET internally so
// the following _POP_TOP_INT becomes _POP_TOP_NOP. Tier 2 only.
tier2 op(_BINARY_OP_ADD_INT_INPLACE, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add);
EXIT_IF(PyStackRef_IsNull(_int_inplace_res));
res = _int_inplace_res;
l = left;
r = right;
INPUTS_DEAD();
}
tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract);
EXIT_IF(PyStackRef_IsNull(_int_inplace_res));
res = _int_inplace_res;
l = left;
r = right;
INPUTS_DEAD();
}
tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply);
EXIT_IF(PyStackRef_IsNull(_int_inplace_res));
res = _int_inplace_res;
l = left;
r = right;
INPUTS_DEAD();
}
tier2 op(_BINARY_OP_ADD_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add);
EXIT_IF(PyStackRef_IsNull(_int_inplace_res));
res = _int_inplace_res;
l = left;
r = right;
INPUTS_DEAD();
}
tier2 op(_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract);
EXIT_IF(PyStackRef_IsNull(_int_inplace_res));
res = _int_inplace_res;
l = left;
r = right;
INPUTS_DEAD();
}
tier2 op(_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, (left, right -- res, l, r)) {
INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply);
EXIT_IF(PyStackRef_IsNull(_int_inplace_res));
res = _int_inplace_res;
l = left;
r = right;
INPUTS_DEAD();
}
op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
EXIT_IF(!PyFloat_CheckExact(left_o));

43
Python/ceval_macros.h
View file

@ -562,3 +562,46 @@ gen_try_set_executing(PyGenObject *gen)
((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \
->ob_fval = _dres; \
} while (0)
// Inplace compact int operation. TARGET is expected to be uniquely
// referenced at the optimizer level, but at runtime it may be a
// cached small int singleton. We check _Py_IsImmortal on TARGET
// to decide whether inplace mutation is safe.
//
// After the macro, _int_inplace_res holds the result (may be NULL
// on allocation failure). On success, TARGET was mutated in place
// and _int_inplace_res is a DUP'd reference to it. On fallback
// (small int target, small int result, or overflow), _int_inplace_res
// is from FUNC (_PyCompactLong_Add etc.).
// FUNC is the fallback function (_PyCompactLong_Add etc.)
#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \
_PyStackRef _int_inplace_res = PyStackRef_NULL; \
do { \
PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \
if (_Py_IsImmortal(target_o)) { \
break; \
} \
assert(_PyObject_IsUniquelyReferenced(target_o)); \
Py_ssize_t left_val = _PyLong_CompactValue( \
(PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \
Py_ssize_t right_val = _PyLong_CompactValue( \
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \
Py_ssize_t result = left_val OP right_val; \
if (!_PY_IS_SMALL_INT(result) \
&& ((twodigits)((stwodigits)result) + PyLong_MASK \
< (twodigits)PyLong_MASK + PyLong_BASE)) \
{ \
_PyLong_SetSignAndDigitCount( \
(PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \
((PyLongObject *)target_o)->long_value.ob_digit[0] = \
(digit)(result < 0 ? -result : result); \
_int_inplace_res = PyStackRef_DUP(TARGET); \
break; \
} \
} while (0); \
if (PyStackRef_IsNull(_int_inplace_res)) { \
_int_inplace_res = FUNC( \
(PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \
(PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \
}

546
Python/executor_cases.c.h generated
View file

@ -4504,6 +4504,552 @@
break;
}
case _BINARY_OP_ADD_INT_INPLACE_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_ADD_INT_INPLACE_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
right = _stack_item_0;
left = stack_pointer[-1];
INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = right;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_ADD_INT_INPLACE_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
INT_INPLACE_OP(left, right, left, +, _PyCompactLong_Add);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = right;
_tos_cache0 = left;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
right = _stack_item_0;
left = stack_pointer[-1];
INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = right;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
INT_INPLACE_OP(left, right, left, -, _PyCompactLong_Subtract);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = right;
_tos_cache0 = left;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
right = _stack_item_0;
left = stack_pointer[-1];
INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = right;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
INT_INPLACE_OP(left, right, left, *, _PyCompactLong_Multiply);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = right;
_tos_cache0 = left;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
right = _stack_item_0;
left = stack_pointer[-1];
INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = right;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
INT_INPLACE_OP(left, right, right, +, _PyCompactLong_Add);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = right;
_tos_cache0 = left;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
right = _stack_item_0;
left = stack_pointer[-1];
INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = right;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
INT_INPLACE_OP(left, right, right, -, _PyCompactLong_Subtract);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = right;
_tos_cache0 = left;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
right = _stack_item_0;
left = stack_pointer[-1];
INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = right;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef right;
_PyStackRef left;
_PyStackRef res;
_PyStackRef l;
_PyStackRef r;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
right = _stack_item_1;
left = _stack_item_0;
INT_INPLACE_OP(left, right, right, *, _PyCompactLong_Multiply);
if (PyStackRef_IsNull(_int_inplace_res)) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = right;
_tos_cache0 = left;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
res = _int_inplace_res;
l = left;
r = right;
_tos_cache2 = r;
_tos_cache1 = l;
_tos_cache0 = res;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _GUARD_NOS_FLOAT_r02: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());

26
Python/optimizer_bytecodes.c
View file

@ -309,21 +309,41 @@ dummy_func(void) {
}
op(_BINARY_OP_ADD_INT, (left, right -- res, l, r)) {
res = sym_new_compact_int(ctx);
if (PyJitRef_IsUnique(left)) {
REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE, 0, 0);
}
else if (PyJitRef_IsUnique(right)) {
REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE_RIGHT, 0, 0);
}
// Result may be a unique compact int or a cached small int
// at runtime. Mark as unique; inplace ops verify at runtime.
res = PyJitRef_MakeUnique(sym_new_compact_int(ctx));
l = left;
r = right;
REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res);
}
op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) {
res = sym_new_compact_int(ctx);
if (PyJitRef_IsUnique(left)) {
REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE, 0, 0);
}
else if (PyJitRef_IsUnique(right)) {
REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, 0, 0);
}
res = PyJitRef_MakeUnique(sym_new_compact_int(ctx));
l = left;
r = right;
REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res);
}
op(_BINARY_OP_MULTIPLY_INT, (left, right -- res, l, r)) {
res = sym_new_compact_int(ctx);
if (PyJitRef_IsUnique(left)) {
REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE, 0, 0);
}
else if (PyJitRef_IsUnique(right)) {
REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, 0, 0);
}
res = PyJitRef_MakeUnique(sym_new_compact_int(ctx));
l = left;
r = right;
REPLACE_OPCODE_IF_EVALUATES_PURE(left, right, res);

120
Python/optimizer_cases.c.h generated
View file

@ -596,7 +596,13 @@
JitOptRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
res = sym_new_compact_int(ctx);
if (PyJitRef_IsUnique(left)) {
REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE, 0, 0);
}
else if (PyJitRef_IsUnique(right)) {
REPLACE_OP(this_instr, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, 0, 0);
}
res = PyJitRef_MakeUnique(sym_new_compact_int(ctx));
l = left;
r = right;
if (
@ -660,7 +666,13 @@
JitOptRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
res = sym_new_compact_int(ctx);
if (PyJitRef_IsUnique(left)) {
REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE, 0, 0);
}
else if (PyJitRef_IsUnique(right)) {
REPLACE_OP(this_instr, _BINARY_OP_ADD_INT_INPLACE_RIGHT, 0, 0);
}
res = PyJitRef_MakeUnique(sym_new_compact_int(ctx));
l = left;
r = right;
if (
@ -724,7 +736,13 @@
JitOptRef r;
right = stack_pointer[-1];
left = stack_pointer[-2];
res = sym_new_compact_int(ctx);
if (PyJitRef_IsUnique(left)) {
REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE, 0, 0);
}
else if (PyJitRef_IsUnique(right)) {
REPLACE_OP(this_instr, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, 0, 0);
}
res = PyJitRef_MakeUnique(sym_new_compact_int(ctx));
l = left;
r = right;
if (
@ -780,6 +798,102 @@
break;
}
case _BINARY_OP_ADD_INT_INPLACE: {
JitOptRef res;
JitOptRef l;
JitOptRef r;
res = sym_new_not_null(ctx);
l = sym_new_not_null(ctx);
r = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-2] = res;
stack_pointer[-1] = l;
stack_pointer[0] = r;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE: {
JitOptRef res;
JitOptRef l;
JitOptRef r;
res = sym_new_not_null(ctx);
l = sym_new_not_null(ctx);
r = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-2] = res;
stack_pointer[-1] = l;
stack_pointer[0] = r;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE: {
JitOptRef res;
JitOptRef l;
JitOptRef r;
res = sym_new_not_null(ctx);
l = sym_new_not_null(ctx);
r = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-2] = res;
stack_pointer[-1] = l;
stack_pointer[0] = r;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _BINARY_OP_ADD_INT_INPLACE_RIGHT: {
JitOptRef res;
JitOptRef l;
JitOptRef r;
res = sym_new_not_null(ctx);
l = sym_new_not_null(ctx);
r = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-2] = res;
stack_pointer[-1] = l;
stack_pointer[0] = r;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT: {
JitOptRef res;
JitOptRef l;
JitOptRef r;
res = sym_new_not_null(ctx);
l = sym_new_not_null(ctx);
r = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-2] = res;
stack_pointer[-1] = l;
stack_pointer[0] = r;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT: {
JitOptRef res;
JitOptRef l;
JitOptRef r;
res = sym_new_not_null(ctx);
l = sym_new_not_null(ctx);
r = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(1);
stack_pointer[-2] = res;
stack_pointer[-1] = l;
stack_pointer[0] = r;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _GUARD_NOS_FLOAT: {
JitOptRef left;
left = stack_pointer[-2];