Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-04 11:00:58 +00:00
Code Issues Projects Releases Packages Wiki Activity

GH-127705: Use _PyStackRefs in the default build. (GH-127875)

Browse source
This commit is contained in:
Mark Shannon 2025-03-10 14:06:56 +00:00 • committed by GitHub
parent 7cc99a54b7
commit 2bef8ea8ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 688 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Include/internal/pycore_ceval.h
View file

@ -342,7 +342,7 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit)
void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value);
PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value);
#ifdef __cplusplus
}

27
Include/internal/pycore_frame.h
View file

@ -148,17 +148,26 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code)
static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest)
{
*dest = *src;
assert(src->stackpointer != NULL);
int stacktop = (int)(src->stackpointer - src->localsplus);
assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus);
dest->stackpointer = dest->localsplus + stacktop;
for (int i = 1; i < stacktop; i++) {
dest->localsplus[i] = src->localsplus[i];
}
dest->f_executable = PyStackRef_MakeHeapSafe(src->f_executable);
// Don't leave a dangling pointer to the old frame when creating generators
// and coroutines:
dest->previous = NULL;
dest->f_funcobj = PyStackRef_MakeHeapSafe(src->f_funcobj);
dest->f_globals = src->f_globals;
dest->f_builtins = src->f_builtins;
dest->f_locals = src->f_locals;
dest->frame_obj = src->frame_obj;
dest->instr_ptr = src->instr_ptr;
#ifdef Py_GIL_DISABLED
dest->tlbc_index = src->tlbc_index;
#endif
assert(src->stackpointer != NULL);
int stacktop = (int)(src->stackpointer - src->localsplus);
assert(stacktop >= 0);
dest->stackpointer = dest->localsplus + stacktop;
for (int i = 0; i < stacktop; i++) {
dest->localsplus[i] = PyStackRef_MakeHeapSafe(src->localsplus[i]);
}
}
#ifdef Py_GIL_DISABLED
@ -393,7 +402,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
PyAPI_FUNC(_PyInterpreterFrame *)
_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func,
PyObject *locals, _PyStackRef const* args,
PyObject *locals, _PyStackRef const *args,
size_t argcount, PyObject *kwnames,
_PyInterpreterFrame *previous);

2
Include/internal/pycore_object.h
View file

@ -82,7 +82,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);
#define _PyObject_HEAD_INIT(type) \
{ \
.ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \
.ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \
.ob_flags = _Py_STATIC_FLAG_BITS, \
.ob_type = (type) \
}
#else

6
Include/internal/pycore_opcode_metadata.h generated
View file

@ -2024,7 +2024,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = {
[BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG },
[BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG },
[BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG },
[BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -2048,9 +2048,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = {
[CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },

386
Include/internal/pycore_stackref.h
View file

@ -60,8 +60,6 @@ extern "C" {
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
typedef union _PyStackRef {
uint64_t index;
} _PyStackRef;
@ -153,6 +151,16 @@ _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
}
#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
static inline void
PyStackRef_XCLOSE(_PyStackRef ref)
{
if (PyStackRef_IsNull(ref)) {
return;
}
PyObject *obj = _Py_stackref_close(ref);
Py_DECREF(obj);
}
static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
@ -162,7 +170,36 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
}
#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref)
extern void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct);
static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
return ref;
}
#define PyStackRef_CLEAR(REF) \
do { \
_PyStackRef *_tmp_op_ptr = &(REF); \
_PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
*_tmp_op_ptr = PyStackRef_NULL; \
PyStackRef_XCLOSE(_tmp_old_op); \
} while (0)
static inline _PyStackRef
_PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber)
{
assert(!_Py_IsImmortal(obj));
return _Py_stackref_create(obj, filename, linenumber);
}
#define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__)
static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
return true;
}
#else
@ -171,12 +208,13 @@ typedef union _PyStackRef {
} _PyStackRef;
#ifdef Py_GIL_DISABLED
#define Py_TAG_DEFERRED (1)
#define Py_TAG_PTR ((uintptr_t)0)
#define Py_TAG_BITS ((uintptr_t)1)
#ifdef Py_GIL_DISABLED
static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
@ -184,6 +222,11 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
// Checks that mask out the deferred bit in the free threading build.
#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None)
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
{
@ -220,6 +263,17 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj)
}
# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
static inline _PyStackRef
PyStackRef_FromPyObjectStealMortal(PyObject *obj)
{
assert(obj != NULL);
assert(!_Py_IsImmortal(obj));
// Make sure we don't take an already tagged value.
assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
return (_PyStackRef){ .bits = (uintptr_t)obj };
}
static inline _PyStackRef
PyStackRef_FromPyObjectNew(PyObject *obj)
{
@ -255,6 +309,13 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj)
} \
} while (0)
static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
(void)destruct;
PyStackRef_CLOSE(ref);
}
static inline _PyStackRef
PyStackRef_DUP(_PyStackRef stackref)
{
@ -269,6 +330,18 @@ PyStackRef_DUP(_PyStackRef stackref)
return stackref;
}
static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
return true;
}
static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
return ref;
}
// Convert a possibly deferred reference to a strong reference.
static inline _PyStackRef
PyStackRef_AsStrongReference(_PyStackRef stackref)
@ -276,55 +349,13 @@ PyStackRef_AsStrongReference(_PyStackRef stackref)
return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref));
}
#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref)
#else // Py_GIL_DISABLED
// With GIL
static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
#define PyStackRef_IsNull(stackref) ((stackref).bits == 0)
#define PyStackRef_True ((_PyStackRef){.bits = (uintptr_t)&_Py_TrueStruct })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
#define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits)
#define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref)
#define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
#define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) })
#define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) })
#define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref))
#define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) _Py_DECREF_SPECIALIZED(PyStackRef_AsPyObjectBorrow(stackref), dealloc)
#endif // Py_GIL_DISABLED
// Check if a stackref is exactly the same as another stackref, including the
// the deferred bit. This can only be used safely if you know that the deferred
// bits of `a` and `b` match.
#define PyStackRef_IsExactly(a, b) \
(assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits)
// Checks that mask out the deferred bit in the free threading build.
#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None)
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
#endif
// Converts a PyStackRef back to a PyObject *, converting the
// stackref to a new reference.
#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))
#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref))
#define PyStackRef_XCLOSE(stackref) \
do { \
_PyStackRef _tmp = (stackref); \
if (!PyStackRef_IsNull(_tmp)) { \
PyStackRef_CLOSE(_tmp); \
} \
} while (0);
#define PyStackRef_CLEAR(op) \
do { \
@ -336,15 +367,250 @@ static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
} \
} while (0)
#define PyStackRef_XCLOSE(stackref) \
do { \
_PyStackRef _tmp = (stackref); \
if (!PyStackRef_IsNull(_tmp)) { \
PyStackRef_CLOSE(_tmp); \
} \
} while (0);
#define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew
#else // Py_GIL_DISABLED
// With GIL
/* References to immortal objects always have their tag bit set to Py_TAG_REFCNT
* as they can (must) have their reclamation deferred */
#define Py_TAG_BITS 1
#define Py_TAG_REFCNT 1
#if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT
# error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT"
#endif
#define BITS_TO_PTR(REF) ((PyObject *)((REF).bits))
#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_BITS)))
#define PyStackRef_NULL_BITS Py_TAG_REFCNT
static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS };
#define PyStackRef_IsNull(ref) ((ref).bits == PyStackRef_NULL_BITS)
#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT })
#define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT))
#define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT))
#define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT))
#ifdef Py_DEBUG
static inline void PyStackRef_CheckValid(_PyStackRef ref) {
assert(ref.bits != 0);
int tag = ref.bits & Py_TAG_BITS;
PyObject *obj = BITS_TO_PTR_MASKED(ref);
switch (tag) {
case 0:
/* Can be immortal if object was made immortal after reference came into existence */
assert(!_Py_IsStaticImmortal(obj));
break;
case Py_TAG_REFCNT:
assert(obj == NULL || _Py_IsImmortal(obj));
break;
default:
assert(0);
}
}
#else
#define PyStackRef_CheckValid(REF) ((void)0)
#endif
#ifdef _WIN32
#define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_BITS) == 0)
#define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED
#else
/* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */
static inline int
PyStackRef_RefcountOnObject(_PyStackRef ref)
{
return (ref.bits & Py_TAG_BITS) == 0;
}
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
{
return BITS_TO_PTR_MASKED(ref);
}
#endif
static inline PyObject *
PyStackRef_AsPyObjectSteal(_PyStackRef ref)
{
if (PyStackRef_RefcountOnObject(ref)) {
return BITS_TO_PTR(ref);
}
else {
return Py_NewRef(BITS_TO_PTR_MASKED(ref));
}
}
static inline _PyStackRef
PyStackRef_FromPyObjectSteal(PyObject *obj)
{
assert(obj != NULL);
#if SIZEOF_VOID_P > 4
unsigned int tag = obj->ob_flags & Py_TAG_BITS;
#else
unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0;
#endif
_PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
PyStackRef_CheckValid(ref);
return ref;
}
static inline _PyStackRef
PyStackRef_FromPyObjectStealMortal(PyObject *obj)
{
assert(obj != NULL);
assert(!_Py_IsImmortal(obj));
_PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) });
PyStackRef_CheckValid(ref);
return ref;
}
// Check if a stackref is exactly the same as another stackref, including the
// the deferred bit. This can only be used safely if you know that the deferred
// bits of `a` and `b` match.
#define PyStackRef_IsExactly(a, b) \
(assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits)
static inline _PyStackRef
_PyStackRef_FromPyObjectNew(PyObject *obj)
{
assert(obj != NULL);
if (_Py_IsImmortal(obj)) {
return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT};
}
Py_INCREF_MORTAL(obj);
_PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
PyStackRef_CheckValid(ref);
return ref;
}
#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))
static inline _PyStackRef
_PyStackRef_FromPyObjectNewMortal(PyObject *obj)
{
assert(obj != NULL);
Py_INCREF_MORTAL(obj);
_PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
PyStackRef_CheckValid(ref);
return ref;
}
#define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj))
/* Create a new reference from an object with an embedded reference count */
static inline _PyStackRef
PyStackRef_FromPyObjectImmortal(PyObject *obj)
{
assert(_Py_IsImmortal(obj));
return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT};
}
/* WARNING: This macro evaluates its argument more than once */
#ifdef _WIN32
#define PyStackRef_DUP(REF) \
(PyStackRef_RefcountOnObject(REF) ? (Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))
#else
static inline _PyStackRef
PyStackRef_DUP(_PyStackRef ref)
{
assert(!PyStackRef_IsNull(ref));
if (PyStackRef_RefcountOnObject(ref)) {
Py_INCREF_MORTAL(BITS_TO_PTR(ref));
}
return ref;
}
#endif
static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
return (ref.bits & Py_TAG_BITS) == 0 || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref));
}
static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
if (PyStackRef_IsHeapSafe(ref)) {
return ref;
}
PyObject *obj = BITS_TO_PTR_MASKED(ref);
Py_INCREF(obj);
ref.bits = (uintptr_t)obj;
PyStackRef_CheckValid(ref);
return ref;
}
#ifdef _WIN32
#define PyStackRef_CLOSE(REF) \
do { \
_PyStackRef _temp = (REF); \
if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \
} while (0)
#else
static inline void
PyStackRef_CLOSE(_PyStackRef ref)
{
assert(!PyStackRef_IsNull(ref));
if (PyStackRef_RefcountOnObject(ref)) {
Py_DECREF_MORTAL(BITS_TO_PTR(ref));
}
}
#endif
static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
assert(!PyStackRef_IsNull(ref));
if (PyStackRef_RefcountOnObject(ref)) {
Py_DECREF_MORTAL_SPECIALIZED(BITS_TO_PTR(ref), destruct);
}
}
#ifdef _WIN32
#define PyStackRef_XCLOSE PyStackRef_CLOSE
#else
static inline void
PyStackRef_XCLOSE(_PyStackRef ref)
{
assert(ref.bits != 0);
if (PyStackRef_RefcountOnObject(ref)) {
assert(!PyStackRef_IsNull(ref));
Py_DECREF_MORTAL(BITS_TO_PTR(ref));
}
}
#endif
#define PyStackRef_CLEAR(REF) \
do { \
_PyStackRef *_tmp_op_ptr = &(REF); \
_PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
*_tmp_op_ptr = PyStackRef_NULL; \
PyStackRef_XCLOSE(_tmp_old_op); \
} while (0)
#endif // Py_GIL_DISABLED
// Note: this is a macro because MSVC (Windows) has trouble inlining it.
#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT)))
#endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref))
// Converts a PyStackRef back to a PyObject *, converting the
// stackref to a new reference.
#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))
// StackRef type checks

4
Include/internal/pycore_uop_metadata.h generated
View file

@ -88,7 +88,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG,
[_BINARY_OP_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG,
[_BINARY_OP_SUBSCR_INIT_CALL] = 0,
@ -242,7 +242,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
[_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,

10
Include/object.h
View file

@ -124,11 +124,13 @@ struct _object {
PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */
struct {
# if PY_BIG_ENDIAN
PY_UINT32_T ob_flags;
PY_UINT32_T ob_refcnt;
uint16_t ob_flags;
uint16_t ob_overflow;
uint32_t ob_refcnt;
# else
PY_UINT32_T ob_refcnt;
PY_UINT32_T ob_flags;
uint32_t ob_refcnt;
uint16_t ob_overflow;
uint16_t ob_flags;
# endif
};
#else

70
Include/refcount.h
View file

@ -19,8 +19,8 @@ immortal. The latter should be the only instances that require
cleanup during runtime finalization.
*/
/* Leave the low bits for refcount overflow for old stable ABI code */
#define _Py_STATICALLY_ALLOCATED_FLAG (1 << 7)
#define _Py_STATICALLY_ALLOCATED_FLAG 4
#define _Py_IMMORTAL_FLAGS 1
#if SIZEOF_VOID_P > 4
/*
@ -43,7 +43,8 @@ be done by checking the bit sign flag in the lower 32 bits.
*/
#define _Py_IMMORTAL_INITIAL_REFCNT (3UL << 30)
#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(_Py_IMMORTAL_INITIAL_REFCNT | (((Py_ssize_t)_Py_STATICALLY_ALLOCATED_FLAG) << 32)))
#define _Py_STATIC_FLAG_BITS ((Py_ssize_t)(_Py_STATICALLY_ALLOCATED_FLAG | _Py_IMMORTAL_FLAGS))
#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT (((Py_ssize_t)_Py_IMMORTAL_INITIAL_REFCNT) | (_Py_STATIC_FLAG_BITS << 48))
#else
/*
@ -114,7 +115,6 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob);
#endif
#endif
static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op)
{
#if defined(Py_GIL_DISABLED)
@ -242,6 +242,18 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *);
PyAPI_FUNC(void) _Py_IncRef(PyObject *);
PyAPI_FUNC(void) _Py_DecRef(PyObject *);
#ifndef Py_GIL_DISABLED
static inline Py_ALWAYS_INLINE void Py_INCREF_MORTAL(PyObject *op)
{
assert(!_Py_IsStaticImmortal(op));
op->ob_refcnt++;
_Py_INCREF_STAT_INC();
#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API)
_Py_INCREF_IncRefTotal();
#endif
}
#endif
static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
{
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
@ -372,6 +384,36 @@ static inline void Py_DECREF(PyObject *op)
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
#elif defined(Py_REF_DEBUG)
static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *op)
{
if (op->ob_refcnt <= 0) {
_Py_NegativeRefcount(filename, lineno, op);
}
_Py_DECREF_STAT_INC();
assert(!_Py_IsStaticImmortal(op));
_Py_DECREF_DecRefTotal();
if (--op->ob_refcnt == 0) {
_Py_Dealloc(op);
}
}
#define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op))
static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int lineno, PyObject *op, destructor destruct)
{
if (op->ob_refcnt <= 0) {
_Py_NegativeRefcount(filename, lineno, op);
}
_Py_DECREF_STAT_INC();
assert(!_Py_IsStaticImmortal(op));
_Py_DECREF_DecRefTotal();
if (--op->ob_refcnt == 0) {
destruct(op);
}
}
#define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) _Py_DECREF_MORTAL_SPECIALIZED(__FILE__, __LINE__, op, destruct)
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
{
#if SIZEOF_VOID_P > 4
@ -396,6 +438,26 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
#else
static inline void Py_DECREF_MORTAL(PyObject *op)
{
assert(!_Py_IsStaticImmortal(op));
_Py_DECREF_STAT_INC();
if (--op->ob_refcnt == 0) {
_Py_Dealloc(op);
}
}
#define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(_PyObject_CAST(op))
static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruct)
{
assert(!_Py_IsStaticImmortal(op));
_Py_DECREF_STAT_INC();
if (--op->ob_refcnt == 0) {
destruct(op);
}
}
#define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct)
static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
{
// Non-limited C API and limited C API for Python 3.9 and older access