gh-140550: Initial implementation of PEP 793 – PyModExport (GH-140556)

Co-authored-by: Victor Stinner <vstinner@python.org>
Co-authored-by: Kumar Aditya <kumaraditya@python.org>
This commit is contained in:
Petr Viktorin 2025-11-05 12:31:42 +01:00 committed by GitHub
parent f2bce51b98
commit 589a03a8ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 1494 additions and 236 deletions

View file

@ -8,7 +8,7 @@
#include "pycore_interp.h" // PyInterpreterState.importlib
#include "pycore_long.h" // _PyLong_GetOne()
#include "pycore_modsupport.h" // _PyModule_CreateInitialized()
#include "pycore_moduleobject.h" // _PyModule_GetDef()
#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull()
#include "pycore_object.h" // _PyType_AllocNoTrack
#include "pycore_pyerrors.h" // _PyErr_FormatFromCause()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
@ -27,6 +27,27 @@ static PyMemberDef module_members[] = {
{0}
};
static void
assert_def_missing_or_redundant(PyModuleObject *m)
{
/* We copy all relevant info into the module object.
* Modules created using a def keep a reference to that (statically
* allocated) def; the info there should match what we have in the module.
*/
#ifndef NDEBUG
if (m->md_token_is_def) {
PyModuleDef *def = (PyModuleDef *)m->md_token;
assert(def);
#define DO_ASSERT(F) assert (def->m_ ## F == m->md_state_ ## F);
DO_ASSERT(size);
DO_ASSERT(traverse);
DO_ASSERT(clear);
DO_ASSERT(free);
#undef DO_ASSERT
}
#endif // NDEBUG
}
PyTypeObject PyModuleDef_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
@ -44,8 +65,14 @@ _PyModule_IsExtension(PyObject *obj)
}
PyModuleObject *module = (PyModuleObject*)obj;
PyModuleDef *def = module->md_def;
return (def != NULL && def->m_methods != NULL);
if (module->md_exec) {
return 1;
}
if (module->md_token_is_def) {
PyModuleDef *def = (PyModuleDef *)module->md_token;
return (module->md_token_is_def && def->m_methods != NULL);
}
return 0;
}
@ -146,10 +173,19 @@ new_module_notrack(PyTypeObject *mt)
m = (PyModuleObject *)_PyType_AllocNoTrack(mt, 0);
if (m == NULL)
return NULL;
m->md_def = NULL;
m->md_state = NULL;
m->md_weaklist = NULL;
m->md_name = NULL;
m->md_token_is_def = false;
#ifdef Py_GIL_DISABLED
m->md_gil = Py_MOD_GIL_USED;
#endif
m->md_state_size = 0;
m->md_state_traverse = NULL;
m->md_state_clear = NULL;
m->md_state_free = NULL;
m->md_exec = NULL;
m->md_token = NULL;
m->md_dict = PyDict_New();
if (m->md_dict == NULL) {
Py_DECREF(m);
@ -264,6 +300,17 @@ PyModule_Create2(PyModuleDef* module, int module_api_version)
return _PyModule_CreateInitialized(module, module_api_version);
}
static void
module_copy_members_from_deflike(
PyModuleObject *md,
PyModuleDef *def_like /* not necessarily a valid Python object */)
{
md->md_state_size = def_like->m_size;
md->md_state_traverse = def_like->m_traverse;
md->md_state_clear = def_like->m_clear;
md->md_state_free = def_like->m_free;
}
PyObject *
_PyModule_CreateInitialized(PyModuleDef* module, int module_api_version)
{
@ -310,15 +357,21 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version)
return NULL;
}
}
m->md_def = module;
m->md_token = module;
m->md_token_is_def = true;
module_copy_members_from_deflike(m, module);
#ifdef Py_GIL_DISABLED
m->md_gil = Py_MOD_GIL_USED;
#endif
return (PyObject*)m;
}
PyObject *
PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version)
static PyObject *
module_from_def_and_spec(
PyModuleDef* def_like, /* not necessarily a valid Python object */
PyObject *spec,
int module_api_version,
PyModuleDef* original_def /* NULL if not defined by a def */)
{
PyModuleDef_Slot* cur_slot;
PyObject *(*create)(PyObject *, PyModuleDef*) = NULL;
@ -331,10 +384,10 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
int has_execution_slots = 0;
const char *name;
int ret;
void *token = NULL;
_Py_modexecfunc m_exec = NULL;
PyInterpreterState *interp = _PyInterpreterState_GET();
PyModuleDef_Init(def);
nameobj = PyObject_GetAttrString(spec, "name");
if (nameobj == NULL) {
return NULL;
@ -348,7 +401,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
goto error;
}
if (def->m_size < 0) {
if (def_like->m_size < 0) {
PyErr_Format(
PyExc_SystemError,
"module %s: m_size may not be negative for multi-phase initialization",
@ -356,7 +409,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
goto error;
}
for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) {
for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) {
// Macro to copy a non-NULL, non-repeatable slot that's unusable with
// PyModuleDef. The destination must be initially NULL.
#define COPY_COMMON_SLOT(SLOT, TYPE, DEST) \
do { \
if (!(TYPE)(cur_slot->value)) { \
PyErr_Format( \
PyExc_SystemError, \
"module %s: " #SLOT " must not be NULL", \
name); \
goto error; \
} \
if (original_def) { \
PyErr_Format( \
PyExc_SystemError, \
"module %s: " #SLOT " used with PyModuleDef", \
name); \
goto error; \
} \
if (DEST) { \
PyErr_Format( \
PyExc_SystemError, \
"module %s has more than one " #SLOT " slot", \
name); \
goto error; \
} \
DEST = (TYPE)(cur_slot->value); \
} while (0); \
/////////////////////////////////////////////////////////////////
switch (cur_slot->slot) {
case Py_mod_create:
if (create) {
@ -370,6 +451,9 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
break;
case Py_mod_exec:
has_execution_slots = 1;
if (!original_def) {
COPY_COMMON_SLOT(Py_mod_exec, _Py_modexecfunc, m_exec);
}
break;
case Py_mod_multiple_interpreters:
if (has_multiple_interpreters_slot) {
@ -398,6 +482,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
goto error;
}
break;
case Py_mod_name:
COPY_COMMON_SLOT(Py_mod_name, char*, def_like->m_name);
break;
case Py_mod_doc:
COPY_COMMON_SLOT(Py_mod_doc, char*, def_like->m_doc);
break;
case Py_mod_state_size:
COPY_COMMON_SLOT(Py_mod_state_size, Py_ssize_t,
def_like->m_size);
break;
case Py_mod_methods:
COPY_COMMON_SLOT(Py_mod_methods, PyMethodDef*,
def_like->m_methods);
break;
case Py_mod_state_traverse:
COPY_COMMON_SLOT(Py_mod_state_traverse, traverseproc,
def_like->m_traverse);
break;
case Py_mod_state_clear:
COPY_COMMON_SLOT(Py_mod_state_clear, inquiry,
def_like->m_clear);
break;
case Py_mod_state_free:
COPY_COMMON_SLOT(Py_mod_state_free, freefunc,
def_like->m_free);
break;
case Py_mod_token:
COPY_COMMON_SLOT(Py_mod_token, void*, token);
break;
default:
assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT);
PyErr_Format(
@ -406,6 +519,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
name, cur_slot->slot);
goto error;
}
#undef COPY_COMMON_SLOT
}
/* By default, multi-phase init modules are expected
@ -429,7 +543,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
}
if (create) {
m = create(spec, def);
m = create(spec, original_def);
if (m == NULL) {
if (!PyErr_Occurred()) {
PyErr_Format(
@ -455,15 +569,27 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
}
if (PyModule_Check(m)) {
((PyModuleObject*)m)->md_state = NULL;
((PyModuleObject*)m)->md_def = def;
PyModuleObject *mod = (PyModuleObject*)m;
mod->md_state = NULL;
module_copy_members_from_deflike(mod, def_like);
if (original_def) {
assert (!token);
mod->md_token = original_def;
mod->md_token_is_def = 1;
}
else {
mod->md_token = token;
}
#ifdef Py_GIL_DISABLED
((PyModuleObject*)m)->md_gil = gil_slot;
mod->md_gil = gil_slot;
#else
(void)gil_slot;
#endif
mod->md_exec = m_exec;
} else {
if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) {
if (def_like->m_size > 0 || def_like->m_traverse || def_like->m_clear
|| def_like->m_free)
{
PyErr_Format(
PyExc_SystemError,
"module %s is not a module object, but requests module state",
@ -478,17 +604,25 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
name);
goto error;
}
if (token) {
PyErr_Format(
PyExc_SystemError,
"module %s specifies a token, but did not create "
"a ModuleType instance",
name);
goto error;
}
}
if (def->m_methods != NULL) {
ret = _add_methods_to_object(m, nameobj, def->m_methods);
if (def_like->m_methods != NULL) {
ret = _add_methods_to_object(m, nameobj, def_like->m_methods);
if (ret != 0) {
goto error;
}
}
if (def->m_doc != NULL) {
ret = PyModule_SetDocString(m, def->m_doc);
if (def_like->m_doc != NULL) {
ret = PyModule_SetDocString(m, def_like->m_doc);
if (ret != 0) {
goto error;
}
@ -503,6 +637,29 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio
return NULL;
}
PyObject *
PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version)
{
PyModuleDef_Init(def);
return module_from_def_and_spec(def, spec, module_api_version, def);
}
PyObject *
PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec)
{
if (!slots) {
PyErr_SetString(
PyExc_SystemError,
"PyModule_FromSlotsAndSpec called with NULL slots");
return NULL;
}
// Fill in enough of a PyModuleDef to pass to common machinery
PyModuleDef def_like = {.m_slots = (PyModuleDef_Slot *)slots};
return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION,
NULL);
}
#ifdef Py_GIL_DISABLED
int
PyUnstable_Module_SetGIL(PyObject *module, void *gil)
@ -516,71 +673,94 @@ PyUnstable_Module_SetGIL(PyObject *module, void *gil)
}
#endif
int
PyModule_ExecDef(PyObject *module, PyModuleDef *def)
static int
run_exec_func(PyObject *module, int (*exec)(PyObject *))
{
PyModuleDef_Slot *cur_slot;
const char *name;
int ret;
name = PyModule_GetName(module);
if (name == NULL) {
int ret = exec(module);
if (ret != 0) {
if (!PyErr_Occurred()) {
PyErr_Format(
PyExc_SystemError,
"execution of %R failed without setting an exception",
module);
}
return -1;
}
if (PyErr_Occurred()) {
_PyErr_FormatFromCause(
PyExc_SystemError,
"execution of module %R raised unreported exception",
module);
return -1;
}
return 0;
}
if (def->m_size >= 0) {
PyModuleObject *md = (PyModuleObject*)module;
static int
alloc_state(PyObject *module)
{
if (!PyModule_Check(module)) {
PyErr_Format(PyExc_TypeError, "expected module, got %T", module);
return -1;
}
PyModuleObject *md = (PyModuleObject*)module;
if (md->md_state_size >= 0) {
if (md->md_state == NULL) {
/* Always set a state pointer; this serves as a marker to skip
* multiple initialization (importlib.reload() is no-op) */
md->md_state = PyMem_Malloc(def->m_size);
md->md_state = PyMem_Malloc(md->md_state_size);
if (!md->md_state) {
PyErr_NoMemory();
return -1;
}
memset(md->md_state, 0, def->m_size);
memset(md->md_state, 0, md->md_state_size);
}
}
return 0;
}
int
PyModule_Exec(PyObject *module)
{
if (alloc_state(module) < 0) {
return -1;
}
PyModuleObject *md = (PyModuleObject*)module;
if (md->md_exec) {
assert(!md->md_token_is_def);
return run_exec_func(module, md->md_exec);
}
PyModuleDef *def = _PyModule_GetDefOrNull(module);
if (def) {
return PyModule_ExecDef(module, def);
}
return 0;
}
int
PyModule_ExecDef(PyObject *module, PyModuleDef *def)
{
PyModuleDef_Slot *cur_slot;
if (alloc_state(module) < 0) {
return -1;
}
assert(PyModule_Check(module));
if (def->m_slots == NULL) {
return 0;
}
for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) {
switch (cur_slot->slot) {
case Py_mod_create:
/* handled in PyModule_FromDefAndSpec2 */
break;
case Py_mod_exec:
ret = ((int (*)(PyObject *))cur_slot->value)(module);
if (ret != 0) {
if (!PyErr_Occurred()) {
PyErr_Format(
PyExc_SystemError,
"execution of module %s failed without setting an exception",
name);
}
return -1;
}
if (PyErr_Occurred()) {
_PyErr_FormatFromCause(
PyExc_SystemError,
"execution of module %s raised unreported exception",
name);
return -1;
}
break;
case Py_mod_multiple_interpreters:
case Py_mod_gil:
case Py_mod_abi:
/* handled in PyModule_FromDefAndSpec2 */
break;
default:
PyErr_Format(
PyExc_SystemError,
"module %s initialized with unknown slot %i",
name, cur_slot->slot);
if (cur_slot->slot == Py_mod_exec) {
int (*func)(PyObject *) = cur_slot->value;
if (run_exec_func(module, func) < 0) {
return -1;
}
continue;
}
}
return 0;
@ -624,6 +804,31 @@ PyModule_GetDict(PyObject *m)
return _PyModule_GetDict(m); // borrowed reference
}
int
PyModule_GetStateSize(PyObject *m, Py_ssize_t *size_p)
{
*size_p = -1;
if (!PyModule_Check(m)) {
PyErr_Format(PyExc_TypeError, "expected module, got %T", m);
return -1;
}
PyModuleObject *mod = (PyModuleObject *)m;
*size_p = mod->md_state_size;
return 0;
}
int
PyModule_GetToken(PyObject *m, void **token_p)
{
*token_p = NULL;
if (!PyModule_Check(m)) {
PyErr_Format(PyExc_TypeError, "expected module, got %T", m);
return -1;
}
*token_p = _PyModule_GetToken(m);
return 0;
}
PyObject*
PyModule_GetNameObject(PyObject *mod)
{
@ -764,7 +969,7 @@ PyModule_GetDef(PyObject* m)
PyErr_BadArgument();
return NULL;
}
return _PyModule_GetDef(m);
return _PyModule_GetDefOrNull(m);
}
void*
@ -888,17 +1093,18 @@ module_dealloc(PyObject *self)
}
FT_CLEAR_WEAKREFS(self, m->md_weaklist);
assert_def_missing_or_redundant(m);
/* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */
if (m->md_def && m->md_def->m_free
&& (m->md_def->m_size <= 0 || m->md_state != NULL))
if (m->md_state_free && (m->md_state_size <= 0 || m->md_state != NULL))
{
m->md_def->m_free(m);
m->md_state_free(m);
}
Py_XDECREF(m->md_dict);
Py_XDECREF(m->md_name);
if (m->md_state != NULL)
if (m->md_state != NULL) {
PyMem_Free(m->md_state);
}
Py_TYPE(m)->tp_free((PyObject *)m);
}
@ -1206,11 +1412,11 @@ module_traverse(PyObject *self, visitproc visit, void *arg)
{
PyModuleObject *m = _PyModule_CAST(self);
assert_def_missing_or_redundant(m);
/* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */
if (m->md_def && m->md_def->m_traverse
&& (m->md_def->m_size <= 0 || m->md_state != NULL))
if (m->md_state_traverse && (m->md_state_size <= 0 || m->md_state != NULL))
{
int res = m->md_def->m_traverse((PyObject*)m, visit, arg);
int res = m->md_state_traverse((PyObject*)m, visit, arg);
if (res)
return res;
}
@ -1224,18 +1430,19 @@ module_clear(PyObject *self)
{
PyModuleObject *m = _PyModule_CAST(self);
assert_def_missing_or_redundant(m);
/* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */
if (m->md_def && m->md_def->m_clear
&& (m->md_def->m_size <= 0 || m->md_state != NULL))
if (m->md_state_clear && (m->md_state_size <= 0 || m->md_state != NULL))
{
int res = m->md_def->m_clear((PyObject*)m);
int res = m->md_state_clear((PyObject*)m);
if (PyErr_Occurred()) {
PyErr_FormatUnraisable("Exception ignored in m_clear of module%s%V",
m->md_name ? " " : "",
m->md_name, "");
}
if (res)
if (res) {
return res;
}
}
Py_CLEAR(m->md_dict);
return 0;