Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-01-06 07:22:09 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-76785: Improved Subinterpreters Compatibility with 3.12 (gh-115424)

Browse source 
For the most part, these changes make is substantially easier to backport subinterpreter-related code to 3.12, especially the related modules (e.g. _xxsubinterpreters). The main motivation is to support releasing a PyPI package with the 3.13 capabilities compiled for 3.12.

A lot of the changes here involve either hiding details behind macros/functions or splitting up some files.
This commit is contained in:
Eric Snow 2024-02-13 14:56:49 -07:00 • committed by GitHub
parent 206f73dc5f
commit 514b1c91b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 857 additions and 719 deletions
Download patch file Download diff file Expand all files Collapse all files

13
Modules/_interpreters_common.h Normal file
View file

@ -0,0 +1,13 @@
#define _RESOLVE_MODINIT_FUNC_NAME(NAME) \
PyInit_ ## NAME
#define RESOLVE_MODINIT_FUNC_NAME(NAME) \
_RESOLVE_MODINIT_FUNC_NAME(NAME)
static int
ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata)
{
//assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE);
return _PyCrossInterpreterData_RegisterClass(cls, getdata);
}

34
Modules/_xxinterpchannelsmodule.c
View file

@ -17,6 +17,8 @@
#include <sched.h> // sched_yield()
#endif
#include "_interpreters_common.h"
/*
This module has the following process-global state:
@ -80,7 +82,9 @@ channel's queue, which are safely managed via the _PyCrossInterpreterData_*()
API.. The module does not create any objects that are shared globally.
*/
#define MODULE_NAME "_xxinterpchannels"
#define MODULE_NAME _xxinterpchannels
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
#define GLOBAL_MALLOC(TYPE) \
@ -101,7 +105,7 @@ static int
register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared,
struct xid_class_registry *classes)
{
int res = _PyCrossInterpreterData_RegisterClass(cls, shared);
int res = ensure_xid_class(cls, shared);
if (res == 0) {
assert(classes->count < MAX_XID_CLASSES);
// The class has refs elsewhere, so we need to incref here.
@ -167,7 +171,7 @@ _get_current_interp(void)
static PyObject *
_get_current_module(void)
{
PyObject *name = PyUnicode_FromString(MODULE_NAME);
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
if (name == NULL) {
return NULL;
}
@ -217,7 +221,7 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
}
#define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \
add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE)
static PyTypeObject *
add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared,
@ -299,7 +303,7 @@ _get_current_module_state(void)
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME " module not imported yet");
MODULE_NAME_STR " module not imported yet");
return NULL;
}
module_state *state = get_module_state(mod);
@ -784,7 +788,7 @@ _channelqueue_clear_interpreter(_channelqueue *queue, int64_t interpid)
while (next != NULL) {
_channelitem *item = next;
next = item->next;
if (item->data->interpid == interpid) {
if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
if (prev == NULL) {
queue->first = item->next;
}
@ -2126,7 +2130,7 @@ static PyStructSequence_Field channel_info_fields[] = {
};
static PyStructSequence_Desc channel_info_desc = {
.name = MODULE_NAME ".ChannelInfo",
.name = MODULE_NAME_STR ".ChannelInfo",
.doc = channel_info_doc,
.fields = channel_info_fields,
.n_in_sequence = 8,
@ -2474,10 +2478,11 @@ struct _channelid_xid {
static PyObject *
_channelid_from_xid(_PyCrossInterpreterData *data)
{
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
struct _channelid_xid *xid = \
(struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
// It might not be imported yet, so we can't use _get_current_module().
PyObject *mod = PyImport_ImportModule(MODULE_NAME);
PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR);
if (mod == NULL) {
return NULL;
}
@ -2530,7 +2535,8 @@ _channelid_shared(PyThreadState *tstate, PyObject *obj,
{
return -1;
}
struct _channelid_xid *xid = (struct _channelid_xid *)data->data;
struct _channelid_xid *xid = \
(struct _channelid_xid *)_PyCrossInterpreterData_DATA(data);
xid->cid = ((channelid *)obj)->cid;
xid->end = ((channelid *)obj)->end;
xid->resolve = ((channelid *)obj)->resolve;
@ -2601,7 +2607,7 @@ static PyType_Slot channelid_typeslots[] = {
};
static PyType_Spec channelid_typespec = {
.name = MODULE_NAME ".ChannelID",
.name = MODULE_NAME_STR ".ChannelID",
.basicsize = sizeof(channelid),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@ -2680,7 +2686,7 @@ _channelend_shared(PyThreadState *tstate, PyObject *obj,
if (res < 0) {
return -1;
}
data->new_object = _channelend_from_xid;
_PyCrossInterpreterData_SET_NEW_OBJECT(data, _channelend_from_xid);
return 0;
}
@ -3379,7 +3385,7 @@ module_free(void *mod)
static struct PyModuleDef moduledef = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = MODULE_NAME,
.m_name = MODULE_NAME_STR,
.m_doc = module_doc,
.m_size = sizeof(module_state),
.m_methods = module_functions,
@ -3390,7 +3396,7 @@ static struct PyModuleDef moduledef = {
};
PyMODINIT_FUNC
PyInit__xxinterpchannels(void)
MODINIT_FUNC_NAME(void)
{
return PyModuleDef_Init(&moduledef);
}

22
Modules/_xxinterpqueuesmodule.c
View file

@ -8,8 +8,12 @@
#include "Python.h"
#include "pycore_crossinterp.h" // struct _xid
#include "_interpreters_common.h"
#define MODULE_NAME "_xxinterpqueues"
#define MODULE_NAME _xxinterpqueues
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
#define GLOBAL_MALLOC(TYPE) \
@ -64,7 +68,7 @@ _get_current_interp(void)
static PyObject *
_get_current_module(void)
{
PyObject *name = PyUnicode_FromString(MODULE_NAME);
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
if (name == NULL) {
return NULL;
}
@ -602,7 +606,7 @@ _queue_clear_interpreter(_queue *queue, int64_t interpid)
while (next != NULL) {
_queueitem *item = next;
next = item->next;
if (item->data->interpid == interpid) {
if (_PyCrossInterpreterData_INTERPID(item->data) == interpid) {
if (prev == NULL) {
queue->items.first = item->next;
}
@ -1062,7 +1066,7 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type)
}
state->queue_type = (PyTypeObject *)Py_NewRef(queue_type);
if (_PyCrossInterpreterData_RegisterClass(queue_type, _queueobj_shared) < 0) {
if (ensure_xid_class(queue_type, _queueobj_shared) < 0) {
return -1;
}
@ -1130,7 +1134,7 @@ _queueid_xid_free(void *data)
static PyObject *
_queueobj_from_xid(_PyCrossInterpreterData *data)
{
int64_t qid = *(int64_t *)data->data;
int64_t qid = *(int64_t *)_PyCrossInterpreterData_DATA(data);
PyObject *qidobj = PyLong_FromLongLong(qid);
if (qidobj == NULL) {
return NULL;
@ -1140,7 +1144,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data)
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME " module not imported yet");
MODULE_NAME_STR " module not imported yet");
return NULL;
}
@ -1181,7 +1185,7 @@ _queueobj_shared(PyThreadState *tstate, PyObject *queueobj,
_PyCrossInterpreterData_Init(data, tstate->interp, raw, NULL,
_queueobj_from_xid);
Py_DECREF(qidobj);
data->free = _queueid_xid_free;
_PyCrossInterpreterData_SET_FREE(data, _queueid_xid_free);
return 0;
}
@ -1670,7 +1674,7 @@ module_free(void *mod)
static struct PyModuleDef moduledef = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = MODULE_NAME,
.m_name = MODULE_NAME_STR,
.m_doc = module_doc,
.m_size = sizeof(module_state),
.m_methods = module_functions,
@ -1681,7 +1685,7 @@ static struct PyModuleDef moduledef = {
};
PyMODINIT_FUNC
PyInit__xxinterpqueues(void)
MODINIT_FUNC_NAME(void)
{
return PyModuleDef_Init(&moduledef);
}

55
Modules/_xxsubinterpretersmodule.c
View file

@ -19,8 +19,12 @@
#include "interpreteridobject.h"
#include "marshal.h" // PyMarshal_ReadObjectFromString()
#include "_interpreters_common.h"
#define MODULE_NAME "_xxsubinterpreters"
#define MODULE_NAME _xxsubinterpreters
#define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME)
#define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME)
static PyInterpreterState *
@ -125,7 +129,7 @@ get_interpid_obj(PyInterpreterState *interp)
static PyObject *
_get_current_module(void)
{
PyObject *name = PyUnicode_FromString(MODULE_NAME);
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
if (name == NULL) {
return NULL;
}
@ -152,16 +156,16 @@ typedef struct {
static PyObject *
xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data)
{
assert(data->data != NULL);
assert(data->obj == NULL);
assert(data->interpid >= 0);
assert(_PyCrossInterpreterData_DATA(data) != NULL);
assert(_PyCrossInterpreterData_OBJ(data) == NULL);
assert(_PyCrossInterpreterData_INTERPID(data) >= 0);
XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject));
if (self == NULL) {
return NULL;
}
PyObject_Init((PyObject *)self, cls);
self->view = (Py_buffer *)data->data;
self->interpid = data->interpid;
self->view = (Py_buffer *)_PyCrossInterpreterData_DATA(data);
self->interpid = _PyCrossInterpreterData_INTERPID(data);
return (PyObject *)self;
}
@ -209,7 +213,7 @@ static PyType_Slot XIBufferViewType_slots[] = {
};
static PyType_Spec XIBufferViewType_spec = {
.name = MODULE_NAME ".CrossInterpreterBufferView",
.name = MODULE_NAME_STR ".CrossInterpreterBufferView",
.basicsize = sizeof(XIBufferViewObject),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
@ -267,8 +271,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state)
*p_state = cls;
// Register XID for the builtin memoryview type.
if (_PyCrossInterpreterData_RegisterClass(
&PyMemoryView_Type, _memoryview_shared) < 0) {
if (ensure_xid_class(&PyMemoryView_Type, _memoryview_shared) < 0) {
return -1;
}
// We don't ever bother un-registering memoryview.
@ -303,7 +306,7 @@ _get_current_module_state(void)
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME " module not imported yet");
MODULE_NAME_STR " module not imported yet");
return NULL;
}
module_state *state = get_module_state(mod);
@ -372,9 +375,7 @@ check_code_object(PyCodeObject *code)
}
// We trust that no code objects under co_consts have unbound cell vars.
if (code->co_executors != NULL
|| code->_co_instrumentation_version > 0)
{
if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) {
return "only basic functions are supported";
}
if (code->_co_monitoring != NULL) {
@ -602,7 +603,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
// Destroy the interpreter.
PyThreadState *tstate = PyThreadState_New(interp);
tstate->_whence = _PyThreadState_WHENCE_INTERP;
_PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
// XXX Possible GILState issues?
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
@ -691,7 +692,7 @@ static PyObject *
interp_set___main___attrs(PyObject *self, PyObject *args)
{
PyObject *id, *updates;
if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME ".set___main___attrs",
if (!PyArg_ParseTuple(args, "OO:" MODULE_NAME_STR ".set___main___attrs",
&id, &updates))
{
return NULL;
@ -856,18 +857,18 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *id, *code;
PyObject *shared = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OO|O:" MODULE_NAME ".exec", kwlist,
"OO|O:" MODULE_NAME_STR ".exec", kwlist,
&id, &code, &shared)) {
return NULL;
}
const char *expected = "a string, a function, or a code object";
if (PyUnicode_Check(code)) {
code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec",
code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec",
"argument 2", expected);
}
else {
code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec",
code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec",
"argument 2", expected);
}
if (code == NULL) {
@ -908,12 +909,12 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *id, *script;
PyObject *shared = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OU|O:" MODULE_NAME ".run_string", kwlist,
"OU|O:" MODULE_NAME_STR ".run_string", kwlist,
&id, &script, &shared)) {
return NULL;
}
script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec",
script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".exec",
"argument 2", "a string");
if (script == NULL) {
return NULL;
@ -934,7 +935,7 @@ PyDoc_STRVAR(run_string_doc,
\n\
Execute the provided string in the identified interpreter.\n\
\n\
(See " MODULE_NAME ".exec().");
(See " MODULE_NAME_STR ".exec().");
static PyObject *
interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
@ -943,12 +944,12 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *id, *func;
PyObject *shared = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"OO|O:" MODULE_NAME ".run_func", kwlist,
"OO|O:" MODULE_NAME_STR ".run_func", kwlist,
&id, &func, &shared)) {
return NULL;
}
PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec",
PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec",
"argument 2",
"a function or a code object");
if (code == NULL) {
@ -972,7 +973,7 @@ Execute the body of the provided function in the identified interpreter.\n\
Code objects are also supported. In both cases, closures and args\n\
are not supported. Methods and other callables are not supported either.\n\
\n\
(See " MODULE_NAME ".exec().");
(See " MODULE_NAME_STR ".exec().");
static PyObject *
@ -1166,7 +1167,7 @@ module_free(void *mod)
static struct PyModuleDef moduledef = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = MODULE_NAME,
.m_name = MODULE_NAME_STR,
.m_doc = module_doc,
.m_size = sizeof(module_state),
.m_methods = module_functions,
@ -1177,7 +1178,7 @@ static struct PyModuleDef moduledef = {
};
PyMODINIT_FUNC
PyInit__xxsubinterpreters(void)
MODINIT_FUNC_NAME(void)
{
return PyModuleDef_Init(&moduledef);
}