Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-12-31 04:23:37 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-115999: Add free-threaded specialization for FOR_ITER (#128798)

Browse source 
Add free-threaded versions of existing specialization for FOR_ITER (list, tuples, fast range iterators and generators), without significantly affecting their thread-safety. (Iterating over shared lists/tuples/ranges should be fine like before. Reusing iterators between threads is not fine, like before. Sharing generators between threads is a recipe for significant crashes, like before.)
This commit is contained in:
T. Wouters 2025-03-12 16:21:46 +01:00 • committed by GitHub
parent db27aee2fe
commit de2f7da77d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 469 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Include/internal/pycore_list.h
View file

@ -8,11 +8,19 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
#ifdef Py_GIL_DISABLED
#include "pycore_stackref.h"
#endif
PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
extern void _PyList_DebugMallocStats(FILE *out);
// _PyList_GetItemRef should be used only when the object is known as a list
// because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does.
extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i);
#ifdef Py_GIL_DISABLED
// Returns -1 in case of races with other threads.
extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *);
#endif
#define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item)

4
Include/internal/pycore_opcode_metadata.h generated
View file

@ -2094,7 +2094,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = {
[FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -2331,7 +2331,7 @@ _PyOpcode_macro_expansion[256] = {
[FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } },
[FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } },
[FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } },
[FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_SIMPLE, 1 } } },
[FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } },
[FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } },
[FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } },
[GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } },

153
Include/internal/pycore_uop_ids.h generated
View file

@ -164,126 +164,127 @@ extern "C" {
#define _ITER_JUMP_RANGE 398
#define _ITER_JUMP_TUPLE 399
#define _ITER_NEXT_LIST 400
#define _ITER_NEXT_RANGE 401
#define _ITER_NEXT_TUPLE 402
#define _JUMP_TO_TOP 403
#define _ITER_NEXT_LIST_TIER_TWO 401
#define _ITER_NEXT_RANGE 402
#define _ITER_NEXT_TUPLE 403
#define _JUMP_TO_TOP 404
#define _LIST_APPEND LIST_APPEND
#define _LIST_EXTEND LIST_EXTEND
#define _LOAD_ATTR 404
#define _LOAD_ATTR_CLASS 405
#define _LOAD_ATTR 405
#define _LOAD_ATTR_CLASS 406
#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
#define _LOAD_ATTR_INSTANCE_VALUE 406
#define _LOAD_ATTR_METHOD_LAZY_DICT 407
#define _LOAD_ATTR_METHOD_NO_DICT 408
#define _LOAD_ATTR_METHOD_WITH_VALUES 409
#define _LOAD_ATTR_MODULE 410
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 411
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 412
#define _LOAD_ATTR_PROPERTY_FRAME 413
#define _LOAD_ATTR_SLOT 414
#define _LOAD_ATTR_WITH_HINT 415
#define _LOAD_ATTR_INSTANCE_VALUE 407
#define _LOAD_ATTR_METHOD_LAZY_DICT 408
#define _LOAD_ATTR_METHOD_NO_DICT 409
#define _LOAD_ATTR_METHOD_WITH_VALUES 410
#define _LOAD_ATTR_MODULE 411
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 412
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 413
#define _LOAD_ATTR_PROPERTY_FRAME 414
#define _LOAD_ATTR_SLOT 415
#define _LOAD_ATTR_WITH_HINT 416
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_BYTECODE 416
#define _LOAD_BYTECODE 417
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
#define _LOAD_CONST LOAD_CONST
#define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL
#define _LOAD_CONST_INLINE 417
#define _LOAD_CONST_INLINE_BORROW 418
#define _LOAD_CONST_INLINE 418
#define _LOAD_CONST_INLINE_BORROW 419
#define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL
#define _LOAD_DEREF LOAD_DEREF
#define _LOAD_FAST 419
#define _LOAD_FAST_0 420
#define _LOAD_FAST_1 421
#define _LOAD_FAST_2 422
#define _LOAD_FAST_3 423
#define _LOAD_FAST_4 424
#define _LOAD_FAST_5 425
#define _LOAD_FAST_6 426
#define _LOAD_FAST_7 427
#define _LOAD_FAST 420
#define _LOAD_FAST_0 421
#define _LOAD_FAST_1 422
#define _LOAD_FAST_2 423
#define _LOAD_FAST_3 424
#define _LOAD_FAST_4 425
#define _LOAD_FAST_5 426
#define _LOAD_FAST_6 427
#define _LOAD_FAST_7 428
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
#define _LOAD_GLOBAL 428
#define _LOAD_GLOBAL_BUILTINS 429
#define _LOAD_GLOBAL_MODULE 430
#define _LOAD_GLOBAL 429
#define _LOAD_GLOBAL_BUILTINS 430
#define _LOAD_GLOBAL_MODULE 431
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SMALL_INT 431
#define _LOAD_SMALL_INT_0 432
#define _LOAD_SMALL_INT_1 433
#define _LOAD_SMALL_INT_2 434
#define _LOAD_SMALL_INT_3 435
#define _LOAD_SMALL_INT 432
#define _LOAD_SMALL_INT_0 433
#define _LOAD_SMALL_INT_1 434
#define _LOAD_SMALL_INT_2 435
#define _LOAD_SMALL_INT_3 436
#define _LOAD_SPECIAL LOAD_SPECIAL
#define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR
#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD
#define _MAKE_CALLARGS_A_TUPLE 436
#define _MAKE_CALLARGS_A_TUPLE 437
#define _MAKE_CELL MAKE_CELL
#define _MAKE_FUNCTION MAKE_FUNCTION
#define _MAKE_WARM 437
#define _MAKE_WARM 438
#define _MAP_ADD MAP_ADD
#define _MATCH_CLASS MATCH_CLASS
#define _MATCH_KEYS MATCH_KEYS
#define _MATCH_MAPPING MATCH_MAPPING
#define _MATCH_SEQUENCE MATCH_SEQUENCE
#define _MAYBE_EXPAND_METHOD 438
#define _MAYBE_EXPAND_METHOD_KW 439
#define _MONITOR_CALL 440
#define _MONITOR_CALL_KW 441
#define _MONITOR_JUMP_BACKWARD 442
#define _MONITOR_RESUME 443
#define _MAYBE_EXPAND_METHOD 439
#define _MAYBE_EXPAND_METHOD_KW 440
#define _MONITOR_CALL 441
#define _MONITOR_CALL_KW 442
#define _MONITOR_JUMP_BACKWARD 443
#define _MONITOR_RESUME 444
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
#define _POP_JUMP_IF_FALSE 444
#define _POP_JUMP_IF_TRUE 445
#define _POP_JUMP_IF_FALSE 445
#define _POP_JUMP_IF_TRUE 446
#define _POP_TOP POP_TOP
#define _POP_TOP_LOAD_CONST_INLINE 446
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 447
#define _POP_TOP_LOAD_CONST_INLINE 447
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 448
#define _PUSH_EXC_INFO PUSH_EXC_INFO
#define _PUSH_FRAME 448
#define _PUSH_FRAME 449
#define _PUSH_NULL PUSH_NULL
#define _PUSH_NULL_CONDITIONAL 449
#define _PY_FRAME_GENERAL 450
#define _PY_FRAME_KW 451
#define _QUICKEN_RESUME 452
#define _REPLACE_WITH_TRUE 453
#define _PUSH_NULL_CONDITIONAL 450
#define _PY_FRAME_GENERAL 451
#define _PY_FRAME_KW 452
#define _QUICKEN_RESUME 453
#define _REPLACE_WITH_TRUE 454
#define _RESUME_CHECK RESUME_CHECK
#define _RETURN_GENERATOR RETURN_GENERATOR
#define _RETURN_VALUE RETURN_VALUE
#define _SAVE_RETURN_OFFSET 454
#define _SEND 455
#define _SEND_GEN_FRAME 456
#define _SAVE_RETURN_OFFSET 455
#define _SEND 456
#define _SEND_GEN_FRAME 457
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
#define _START_EXECUTOR 457
#define _STORE_ATTR 458
#define _STORE_ATTR_INSTANCE_VALUE 459
#define _STORE_ATTR_SLOT 460
#define _STORE_ATTR_WITH_HINT 461
#define _START_EXECUTOR 458
#define _STORE_ATTR 459
#define _STORE_ATTR_INSTANCE_VALUE 460
#define _STORE_ATTR_SLOT 461
#define _STORE_ATTR_WITH_HINT 462
#define _STORE_DEREF STORE_DEREF
#define _STORE_FAST 462
#define _STORE_FAST_0 463
#define _STORE_FAST_1 464
#define _STORE_FAST_2 465
#define _STORE_FAST_3 466
#define _STORE_FAST_4 467
#define _STORE_FAST_5 468
#define _STORE_FAST_6 469
#define _STORE_FAST_7 470
#define _STORE_FAST 463
#define _STORE_FAST_0 464
#define _STORE_FAST_1 465
#define _STORE_FAST_2 466
#define _STORE_FAST_3 467
#define _STORE_FAST_4 468
#define _STORE_FAST_5 469
#define _STORE_FAST_6 470
#define _STORE_FAST_7 471
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE 471
#define _STORE_SUBSCR 472
#define _STORE_SLICE 472
#define _STORE_SUBSCR 473
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
#define _TIER2_RESUME_CHECK 473
#define _TO_BOOL 474
#define _TIER2_RESUME_CHECK 474
#define _TO_BOOL 475
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
@ -293,13 +294,13 @@ extern "C" {
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
#define _UNPACK_SEQUENCE 475
#define _UNPACK_SEQUENCE 476
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define _YIELD_VALUE YIELD_VALUE
#define MAX_UOP_ID 475
#define MAX_UOP_ID 476
#ifdef __cplusplus
}

6
Include/internal/pycore_uop_metadata.h generated
View file

@ -185,7 +185,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_ITER_CHECK_LIST] = HAS_EXIT_FLAG,
[_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG,
[_ITER_NEXT_LIST] = 0,
[_ITER_NEXT_LIST_TIER_TWO] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG,
[_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG,
[_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG,
[_ITER_NEXT_TUPLE] = 0,
@ -428,7 +428,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_ITER_CHECK_LIST] = "_ITER_CHECK_LIST",
[_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE",
[_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE",
[_ITER_NEXT_LIST] = "_ITER_NEXT_LIST",
[_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO",
[_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE",
[_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE",
[_JUMP_TO_TOP] = "_JUMP_TO_TOP",
@ -889,7 +889,7 @@ int _PyUop_num_popped(int opcode, int oparg)
return 0;
case _GUARD_NOT_EXHAUSTED_LIST:
return 0;
case _ITER_NEXT_LIST:
case _ITER_NEXT_LIST_TIER_TWO:
return 0;
case _ITER_CHECK_TUPLE:
return 0;

39
Lib/test/test_opcache.py
View file

@ -1802,5 +1802,44 @@ def unused(): pass
self.assert_specialized(load_const, "LOAD_CONST_MORTAL")
self.assert_no_opcode(load_const, "LOAD_CONST")
@cpython_only
@requires_specialization_ft
def test_for_iter(self):
L = list(range(10))
def for_iter_list():
for i in L:
self.assertIn(i, L)
for_iter_list()
self.assert_specialized(for_iter_list, "FOR_ITER_LIST")
self.assert_no_opcode(for_iter_list, "FOR_ITER")
t = tuple(range(10))
def for_iter_tuple():
for i in t:
self.assertIn(i, t)
for_iter_tuple()
self.assert_specialized(for_iter_tuple, "FOR_ITER_TUPLE")
self.assert_no_opcode(for_iter_tuple, "FOR_ITER")
r = range(10)
def for_iter_range():
for i in r:
self.assertIn(i, r)
for_iter_range()
self.assert_specialized(for_iter_range, "FOR_ITER_RANGE")
self.assert_no_opcode(for_iter_range, "FOR_ITER")
def for_iter_generator():
for i in (i for i in range(10)):
i + 1
for_iter_generator()
self.assert_specialized(for_iter_generator, "FOR_ITER_GEN")
self.assert_no_opcode(for_iter_generator, "FOR_ITER")
if __name__ == "__main__":
unittest.main()

26
Objects/listobject.c
View file

@ -415,6 +415,32 @@ _PyList_GetItemRef(PyListObject *list, Py_ssize_t i)
return list_get_item_ref(list, i);
}
#ifdef Py_GIL_DISABLED
int
_PyList_GetItemRefNoLock(PyListObject *list, Py_ssize_t i, _PyStackRef *result)
{
assert(_Py_IsOwnedByCurrentThread((PyObject *)list) ||
_PyObject_GC_IS_SHARED(list));
if (!valid_index(i, PyList_GET_SIZE(list))) {
return 0;
}
PyObject **ob_item = _Py_atomic_load_ptr(&list->ob_item);
if (ob_item == NULL) {
return 0;
}
Py_ssize_t cap = list_capacity(ob_item);
assert(cap != -1);
if (!valid_index(i, cap)) {
return 0;
}
PyObject *obj = _Py_atomic_load_ptr(&ob_item[i]);
if (obj == NULL || !_Py_TryIncrefCompareStackRef(&ob_item[i], obj, result)) {
return -1;
}
return 1;
}
#endif
int
PyList_SetItem(PyObject *op, Py_ssize_t i,
PyObject *newitem)

113
Python/bytecodes.c
View file

@ -3005,7 +3005,7 @@ dummy_func(
};
specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) {
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ForIter(iter, next_instr, oparg);
@ -3013,7 +3013,7 @@ dummy_func(
}
OPCODE_DEFERRED_INC(FOR_ITER);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
replaced op(_FOR_ITER, (iter -- iter, next)) {
@ -3091,31 +3091,46 @@ dummy_func(
op(_ITER_CHECK_LIST, (iter -- iter)) {
EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type);
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type);
#ifdef Py_GIL_DISABLED
EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o));
_PyListIterObject *it = (_PyListIterObject *)iter_o;
EXIT_IF(!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
!_PyObject_GC_IS_SHARED(it->it_seq));
#endif
}
replaced op(_ITER_JUMP_LIST, (iter -- iter)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
// For free-threaded Python, the loop exit can happen at any point during
// item retrieval, so it doesn't make much sense to check and jump
// separately before item retrieval. Any length check we do here can be
// invalid by the time we actually try to fetch the item.
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
(void)iter_o;
#else
_PyListIterObject *it = (_PyListIterObject *)iter_o;
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
it->it_index = -1;
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
#endif
/* Jump forward oparg, then skip following END_FOR instruction */
JUMPBY(oparg + 1);
DISPATCH();
}
#endif
}
// Only used by Tier 2
op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) {
#ifndef Py_GIL_DISABLED
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
@ -3125,16 +3140,62 @@ dummy_func(
it->it_index = -1;
EXIT_IF(1);
}
#endif
}
op(_ITER_NEXT_LIST, (iter -- iter, next)) {
replaced op(_ITER_NEXT_LIST, (iter -- iter, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
_PyObject_GC_IS_SHARED(seq));
STAT_INC(FOR_ITER, hit);
int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
// A negative result means we lost a race with another thread
// and we need to take the slow path.
DEOPT_IF(result < 0);
if (result == 0) {
it->it_index = -1;
/* Jump forward oparg, then skip following END_FOR instruction */
JUMPBY(oparg + 1);
DISPATCH();
}
it->it_index++;
#else
assert(it->it_index < PyList_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
#endif
}
// Only used by Tier 2
op(_ITER_NEXT_LIST_TIER_TWO, (iter -- iter, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
_PyObject_GC_IS_SHARED(seq));
STAT_INC(FOR_ITER, hit);
int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
// A negative result means we lost a race with another thread
// and we need to take the slow path.
EXIT_IF(result < 0);
if (result == 0) {
it->it_index = -1;
EXIT_IF(1);
}
it->it_index++;
#else
assert(it->it_index < PyList_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
#endif
}
macro(FOR_ITER_LIST) =
@ -3144,20 +3205,30 @@ dummy_func(
_ITER_NEXT_LIST;
op(_ITER_CHECK_TUPLE, (iter -- iter)) {
EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type);
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type);
#ifdef Py_GIL_DISABLED
EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o));
#endif
}
replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
(void)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
STAT_INC(FOR_ITER, hit);
PyTupleObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) {
if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) {
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
#endif
/* Jump forward oparg, then skip following END_FOR instruction */
JUMPBY(oparg + 1);
DISPATCH();
@ -3169,6 +3240,9 @@ dummy_func(
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
PyTupleObject *seq = it->it_seq;
EXIT_IF(seq == NULL);
EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq));
@ -3179,6 +3253,9 @@ dummy_func(
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
@ -3193,11 +3270,17 @@ dummy_func(
op(_ITER_CHECK_RANGE, (iter -- iter)) {
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
EXIT_IF(!_PyObject_IsUniquelyReferenced((PyObject *)r));
#endif
}
replaced op(_ITER_JUMP_RANGE, (iter -- iter)) {
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced((PyObject *)r));
#endif
STAT_INC(FOR_ITER, hit);
if (r->len <= 0) {
// Jump over END_FOR instruction.
@ -3216,6 +3299,9 @@ dummy_func(
op(_ITER_NEXT_RANGE, (iter -- iter, next)) {
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced((PyObject *)r));
#endif
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
@ -3234,6 +3320,13 @@ dummy_func(
op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) {
PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter);
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type);
#ifdef Py_GIL_DISABLED
// Since generators can't be used by multiple threads anyway we
// don't need to deopt here, but this lets us work on making
// generators thread-safe without necessarily having to
// specialize them thread-safely as well.
DEOPT_IF(!_PyObject_IsUniquelyReferenced((PyObject *)gen));
#endif
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING);
STAT_INC(FOR_ITER, hit);
gen_frame = &gen->gi_iframe;

79
Python/executor_cases.c.h generated
View file

@ -4171,10 +4171,23 @@
case _ITER_CHECK_LIST: {
_PyStackRef iter;
iter = stack_pointer[-1];
if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o) != &PyListIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced(iter_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
_PyListIterObject *it = (_PyListIterObject *)iter_o;
if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
!_PyObject_GC_IS_SHARED(it->it_seq)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#endif
break;
}
@ -4183,6 +4196,7 @@
case _GUARD_NOT_EXHAUSTED_LIST: {
_PyStackRef iter;
iter = stack_pointer[-1];
#ifndef Py_GIL_DISABLED
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
@ -4198,10 +4212,13 @@
JUMP_TO_JUMP_TARGET();
}
}
#endif
break;
}
case _ITER_NEXT_LIST: {
/* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */
case _ITER_NEXT_LIST_TIER_TWO: {
_PyStackRef iter;
_PyStackRef next;
iter = stack_pointer[-1];
@ -4210,8 +4227,32 @@
assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
_PyObject_GC_IS_SHARED(seq));
STAT_INC(FOR_ITER, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
stack_pointer = _PyFrame_GetStackPointer(frame);
// A negative result means we lost a race with another thread
// and we need to take the slow path.
if (result < 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (result == 0) {
it->it_index = -1;
if (1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
it->it_index++;
#else
assert(it->it_index < PyList_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
#endif
stack_pointer[0] = next;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@ -4221,10 +4262,17 @@
case _ITER_CHECK_TUPLE: {
_PyStackRef iter;
iter = stack_pointer[-1];
if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o) != &PyTupleIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced(iter_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#endif
break;
}
@ -4236,6 +4284,9 @@
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
PyTupleObject *seq = it->it_seq;
if (seq == NULL) {
UOP_STAT_INC(uopcode, miss);
@ -4256,6 +4307,9 @@
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));
@ -4273,6 +4327,12 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#endif
break;
}
@ -4296,6 +4356,9 @@
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced((PyObject *)r));
#endif
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
@ -4321,6 +4384,16 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
// Since generators can't be used by multiple threads anyway we
// don't need to deopt here, but this lets us work on making
// generators thread-safe without necessarily having to
// specialize them thread-safely as well.
if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#endif
if (gen->gi_frame_state >= FRAME_EXECUTING) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();

105
Python/generated_cases.c.h generated
View file

@ -5434,7 +5434,7 @@
iter = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -5444,7 +5444,7 @@
}
OPCODE_DEFERRED_INC(FOR_ITER);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
// _FOR_ITER
{
@ -5514,6 +5514,17 @@
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#ifdef Py_GIL_DISABLED
// Since generators can't be used by multiple threads anyway we
// don't need to deopt here, but this lets us work on making
// generators thread-safe without necessarily having to
// specialize them thread-safely as well.
if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#endif
if (gen->gi_frame_state >= FRAME_EXECUTING) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
@ -5565,33 +5576,55 @@
// _ITER_CHECK_LIST
{
iter = stack_pointer[-1];
if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyListIter_Type) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o) != &PyListIter_Type) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced(iter_o)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
_PyListIterObject *it = (_PyListIterObject *)iter_o;
if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) ||
!_PyObject_GC_IS_SHARED(it->it_seq)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#endif
}
// _ITER_JUMP_LIST
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyListIterObject *it = (_PyListIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyListIter_Type);
// For free-threaded Python, the loop exit can happen at any point during
// item retrieval, so it doesn't make much sense to check and jump
// separately before item retrieval. Any length check we do here can be
// invalid by the time we actually try to fetch the item.
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
(void)iter_o;
#else
_PyListIterObject *it = (_PyListIterObject *)iter_o;
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
it->it_index = -1;
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
#endif
/* Jump forward oparg, then skip following END_FOR instruction */
JUMPBY(oparg + 1);
DISPATCH();
}
#endif
}
// _ITER_NEXT_LIST
{
@ -5600,8 +5633,32 @@
assert(Py_TYPE(iter_o) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) ||
_PyObject_GC_IS_SHARED(seq));
STAT_INC(FOR_ITER, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next);
stack_pointer = _PyFrame_GetStackPointer(frame);
// A negative result means we lost a race with another thread
// and we need to take the slow path.
if (result < 0) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
if (result == 0) {
it->it_index = -1;
/* Jump forward oparg, then skip following END_FOR instruction */
JUMPBY(oparg + 1);
DISPATCH();
}
it->it_index++;
#else
assert(it->it_index < PyList_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++));
#endif
}
stack_pointer[0] = next;
stack_pointer += 1;
@ -5632,11 +5689,21 @@
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#endif
}
// _ITER_JUMP_RANGE
{
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced((PyObject *)r));
#endif
STAT_INC(FOR_ITER, hit);
if (r->len <= 0) {
// Jump over END_FOR instruction.
@ -5648,6 +5715,9 @@
{
_PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter);
assert(Py_TYPE(r) == &PyRangeIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced((PyObject *)r));
#endif
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
@ -5681,26 +5751,40 @@
// _ITER_CHECK_TUPLE
{
iter = stack_pointer[-1];
if (Py_TYPE(PyStackRef_AsPyObjectBorrow(iter)) != &PyTupleIter_Type) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
if (Py_TYPE(iter_o) != &PyTupleIter_Type) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#ifdef Py_GIL_DISABLED
if (!_PyObject_IsUniquelyReferenced(iter_o)) {
UPDATE_MISS_STATS(FOR_ITER);
assert(_PyOpcode_Deopt[opcode] == (FOR_ITER));
JUMP_TO_PREDICTED(FOR_ITER);
}
#endif
}
// _ITER_JUMP_TUPLE
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
(void)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
STAT_INC(FOR_ITER, hit);
PyTupleObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) {
if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) {
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
_PyFrame_SetStackPointer(frame, stack_pointer);
Py_DECREF(seq);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
#endif
/* Jump forward oparg, then skip following END_FOR instruction */
JUMPBY(oparg + 1);
DISPATCH();
@ -5712,6 +5796,9 @@
_PyTupleIterObject *it = (_PyTupleIterObject *)iter_o;
assert(Py_TYPE(iter_o) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
#ifdef Py_GIL_DISABLED
assert(_PyObject_IsUniquelyReferenced(iter_o));
#endif
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++));

1
Python/optimizer.c
View file

@ -363,6 +363,7 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = {
[_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST,
[_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE,
[_FOR_ITER] = _FOR_ITER_TIER_TWO,
[_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO,
};
static const uint8_t

4
Python/optimizer_cases.c.h generated
View file

@ -1458,7 +1458,9 @@
break;
}
case _ITER_NEXT_LIST: {
/* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */
case _ITER_NEXT_LIST_TIER_TWO: {
JitOptSymbol *next;
next = sym_new_not_null(ctx);
stack_pointer[0] = next;

53
Python/specialize.c
View file

@ -2826,45 +2826,56 @@ int
void
_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg)
{
assert(ENABLE_SPECIALIZATION);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER);
_PyForIterCache *cache = (_PyForIterCache *)(instr + 1);
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
PyTypeObject *tp = Py_TYPE(iter_o);
#ifdef Py_GIL_DISABLED
// Only specialize for uniquely referenced iterators, so that we know
// they're only referenced by this one thread. This is more limiting
// than we need (even `it = iter(mylist); for item in it:` won't get
// specialized) but we don't have a way to check whether we're the only
// _thread_ who has access to the object.
if (!_PyObject_IsUniquelyReferenced(iter_o))
goto failure;
#endif
if (tp == &PyListIter_Type) {
instr->op.code = FOR_ITER_LIST;
goto success;
#ifdef Py_GIL_DISABLED
_PyListIterObject *it = (_PyListIterObject *)iter_o;
if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) &&
!_PyObject_GC_IS_SHARED(it->it_seq)) {
// Maybe this should just set GC_IS_SHARED in a critical
// section, instead of leaving it to the first iteration?
goto failure;
}
#endif
specialize(instr, FOR_ITER_LIST);
return;
}
else if (tp == &PyTupleIter_Type) {
instr->op.code = FOR_ITER_TUPLE;
goto success;
specialize(instr, FOR_ITER_TUPLE);
return;
}
else if (tp == &PyRangeIter_Type) {
instr->op.code = FOR_ITER_RANGE;
goto success;
specialize(instr, FOR_ITER_RANGE);
return;
}
else if (tp == &PyGen_Type && oparg <= SHRT_MAX) {
// Generators are very much not thread-safe, so don't worry about
// the specialization not being thread-safe.
assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR ||
instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR
);
/* Don't specialize if PEP 523 is active */
if (_PyInterpreterState_GET()->eval_frame) {
SPECIALIZATION_FAIL(FOR_ITER, SPEC_FAIL_OTHER);
if (_PyInterpreterState_GET()->eval_frame)
goto failure;
specialize(instr, FOR_ITER_GEN);
return;
}
instr->op.code = FOR_ITER_GEN;
goto success;
}
failure:
SPECIALIZATION_FAIL(FOR_ITER,
_PySpecialization_ClassifyIterator(iter_o));
failure:
STAT_INC(FOR_ITER, failure);
instr->op.code = FOR_ITER;
cache->counter = adaptive_counter_backoff(cache->counter);
return;
success:
STAT_INC(FOR_ITER, success);
cache->counter = adaptive_counter_cooldown();
unspecialize(instr);
}
void

3
Tools/cases_generator/analyzer.py
View file

@ -644,11 +644,13 @@ def has_error_without_pop(op: parser.CodeDef) -> bool:
"_PyLong_IsNonNegativeCompact",
"_PyLong_IsZero",
"_PyManagedDictPointer_IsValues",
"_PyObject_GC_IS_SHARED",
"_PyObject_GC_IS_TRACKED",
"_PyObject_GC_MAY_BE_TRACKED",
"_PyObject_GC_TRACK",
"_PyObject_GetManagedDict",
"_PyObject_InlineValues",
"_PyObject_IsUniquelyReferenced",
"_PyObject_ManagedDictPointer",
"_PyThreadState_HasStackSpace",
"_PyTuple_FromStackRefStealOnSuccess",
@ -661,6 +663,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool:
"_Py_DECREF_NO_DEALLOC",
"_Py_ID",
"_Py_IsImmortal",
"_Py_IsOwnedByCurrentThread",
"_Py_LeaveRecursiveCallPy",
"_Py_LeaveRecursiveCallTstate",
"_Py_NewRef",