gh-111506: Add _Py_OPAQUE_PYOBJECT to hide PyObject layout & related API (GH-136505)

Allow Py_LIMITED_API for (Py_GIL_DISABLED && _Py_OPAQUE_PYOBJECT)


API that's removed when _Py_OPAQUE_PYOBJECT is defined:

    - PyObject_HEAD
    - _PyObject_EXTRA_INIT
    - PyObject_HEAD_INIT
    - PyObject_VAR_HEAD
    - struct _object (i.e. PyObject) (opaque)
    - struct PyVarObject (opaque)
    - Py_SIZE
    - Py_SET_TYPE
    - Py_SET_SIZE
    - PyModuleDef_Base (opaque)
    - PyModuleDef_HEAD_INIT
    - PyModuleDef (opaque)
    - _Py_IsImmortal
    - _Py_IsStaticImmortal

Note that the `_Py_IsImmortal` removal (and a few other issues)
 means _Py_OPAQUE_PYOBJECT only works with limited
API 3.14+ now.


Co-authored-by: Victor Stinner <vstinner@python.org>

Include/Python.h

@@ -45,19 +45,19 @@
 #  endif
 #endif
 
-// gh-111506: The free-threaded build is not compatible with the limited API
-// or the stable ABI.
-#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED)
-#  error "The limited API is not currently supported in the free-threaded build"
-#endif
+#if defined(Py_GIL_DISABLED)
+#  if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT)
+#    error "Py_LIMITED_API is not currently supported in the free-threaded build"
+#  endif
 
-#if defined(Py_GIL_DISABLED) && defined(_MSC_VER)
-#  include <intrin.h>             // __readgsqword()
-#endif
+#  if defined(_MSC_VER)
+#    include <intrin.h>             // __readgsqword()
+#  endif
 
-#if defined(Py_GIL_DISABLED) && defined(__MINGW32__)
-#  include <intrin.h>             // __readgsqword()
-#endif
+#  if defined(__MINGW32__)
+#    include <intrin.h>             // __readgsqword()
+#  endif
+#endif // Py_GIL_DISABLED
 
 // Include Python header files
 #include "pyport.h"

Include/moduleobject.h

@@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) PyModuleDef_Init(PyModuleDef*);
 PyAPI_DATA(PyTypeObject) PyModuleDef_Type;
 #endif
 
+#ifndef _Py_OPAQUE_PYOBJECT
 typedef struct PyModuleDef_Base {
   PyObject_HEAD
   /* The function used to re-initialize the module.
@@ -63,6 +64,7 @@ typedef struct PyModuleDef_Base {
     0,        /* m_index */      \
     _Py_NULL, /* m_copy */       \
   }
+#endif  // _Py_OPAQUE_PYOBJECT
 
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
 /* New in 3.5 */
@@ -104,6 +106,8 @@ struct PyModuleDef_Slot {
 PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil);
 #endif
 
+
+#ifndef _Py_OPAQUE_PYOBJECT
 struct PyModuleDef {
   PyModuleDef_Base m_base;
   const char* m_name;
@@ -115,6 +119,7 @@ struct PyModuleDef {
   inquiry m_clear;
   freefunc m_free;
 };
+#endif  // _Py_OPAQUE_PYOBJECT
 
 #ifdef __cplusplus
 }

Include/object.h

@@ -56,6 +56,11 @@ whose size is determined when the object is allocated.
 #  define Py_REF_DEBUG
 #endif
 
+#if defined(_Py_OPAQUE_PYOBJECT) && !defined(Py_LIMITED_API)
+#   error "_Py_OPAQUE_PYOBJECT only makes sense with Py_LIMITED_API"
+#endif
+
+#ifndef _Py_OPAQUE_PYOBJECT
 /* PyObject_HEAD defines the initial segment of every PyObject. */
 #define PyObject_HEAD                   PyObject ob_base;
 
@@ -99,6 +104,8 @@ whose size is determined when the object is allocated.
  * not necessarily a byte count.
  */
 #define PyObject_VAR_HEAD      PyVarObject ob_base;
+#endif // !defined(_Py_OPAQUE_PYOBJECT)
+
 #define Py_INVALID_SIZE (Py_ssize_t)-1
 
 /* PyObjects are given a minimum alignment so that the least significant bits
@@ -112,7 +119,9 @@ whose size is determined when the object is allocated.
  * by hand.  Similarly every pointer to a variable-size Python object can,
  * in addition, be cast to PyVarObject*.
  */
-#ifndef Py_GIL_DISABLED
+#ifdef _Py_OPAQUE_PYOBJECT
+  /* PyObject is opaque */
+#elif !defined(Py_GIL_DISABLED)
 struct _object {
 #if (defined(__GNUC__) || defined(__clang__)) \
         && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L)
@@ -168,15 +177,18 @@ struct _object {
     Py_ssize_t ob_ref_shared;   // shared (atomic) reference count
     PyTypeObject *ob_type;
 };
-#endif
+#endif // !defined(_Py_OPAQUE_PYOBJECT)
 
 /* Cast argument to PyObject* type. */
 #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op))
 
-typedef struct {
+#ifndef _Py_OPAQUE_PYOBJECT
+struct PyVarObject {
     PyObject ob_base;
     Py_ssize_t ob_size; /* Number of items in variable part */
-} PyVarObject;
+};
+#endif
+typedef struct PyVarObject PyVarObject;
 
 /* Cast argument to PyVarObject* type. */
 #define _PyVarObject_CAST(op) _Py_CAST(PyVarObject*, (op))
@@ -286,6 +298,7 @@ PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob);
 PyAPI_DATA(PyTypeObject) PyLong_Type;
 PyAPI_DATA(PyTypeObject) PyBool_Type;
 
+#ifndef _Py_OPAQUE_PYOBJECT
 // bpo-39573: The Py_SET_SIZE() function must be used to set an object size.
 static inline Py_ssize_t Py_SIZE(PyObject *ob) {
     assert(Py_TYPE(ob) != &PyLong_Type);
@@ -295,6 +308,7 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) {
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
 #  define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob))
 #endif
+#endif // !defined(_Py_OPAQUE_PYOBJECT)
 
 static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) {
     return Py_TYPE(ob) == type;
@@ -304,6 +318,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) {
 #endif
 
 
+#ifndef _Py_OPAQUE_PYOBJECT
 static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) {
     ob->ob_type = type;
 }
@@ -323,6 +338,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) {
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
 #  define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size))
 #endif
+#endif // !defined(_Py_OPAQUE_PYOBJECT)
 
 
 /*

Include/refcount.h

@@ -117,6 +117,7 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob);
     #endif
 #endif
 
+#ifndef _Py_OPAQUE_PYOBJECT
 static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op)
 {
 #if defined(Py_GIL_DISABLED)
@@ -140,6 +141,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op)
 #endif
 }
 #define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op))
+#endif // !defined(_Py_OPAQUE_PYOBJECT)
 
 // Py_SET_REFCNT() implementation for stable ABI
 PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt);

Lib/test/test_cext/__init__.py

@@ -12,7 +12,10 @@
 from test import support
 
 
-SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c')
+SOURCES = [
+    os.path.join(os.path.dirname(__file__), 'extension.c'),
+    os.path.join(os.path.dirname(__file__), 'create_moduledef.c'),
+]
 SETUP = os.path.join(os.path.dirname(__file__), 'setup.py')
 
 
@@ -35,17 +38,22 @@ class BaseTests:
     def test_build(self):
         self.check_build('_test_cext')
 
-    def check_build(self, extension_name, std=None, limited=False):
+    def check_build(self, extension_name, std=None, limited=False,
+                    opaque_pyobject=False):
         venv_dir = 'env'
         with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
             self._check_build(extension_name, python_exe,
-                              std=std, limited=limited)
+                              std=std, limited=limited,
+                              opaque_pyobject=opaque_pyobject)
 
-    def _check_build(self, extension_name, python_exe, std, limited):
+    def _check_build(self, extension_name, python_exe, std, limited,
+                     opaque_pyobject):
         pkg_dir = 'pkg'
         os.mkdir(pkg_dir)
         shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP)))
-        shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE)))
+        for source in SOURCES:
+            dest = os.path.join(pkg_dir, os.path.basename(source))
+            shutil.copy(source, dest)
 
         def run_cmd(operation, cmd):
             env = os.environ.copy()
@@ -53,6 +61,8 @@ def run_cmd(operation, cmd):
                 env['CPYTHON_TEST_STD'] = std
             if limited:
                 env['CPYTHON_TEST_LIMITED'] = '1'
+            if opaque_pyobject:
+                env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1'
             env['CPYTHON_TEST_EXT_NAME'] = extension_name
             env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
             if support.verbose:
@@ -107,6 +117,11 @@ def test_build_limited_c11(self):
     def test_build_c11(self):
         self.check_build('_test_c11_cext', std='c11')
 
+    def test_build_opaque_pyobject(self):
+        # Test with _Py_OPAQUE_PYOBJECT
+        self.check_build('_test_limited_opaque_cext', limited=True,
+                         opaque_pyobject=True)
+
     @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
     def test_build_c99(self):
         # In public docs, we say C API is compatible with C11. However,

Lib/test/test_cext/create_moduledef.c

@@ -0,0 +1,29 @@
+// Workaround for testing _Py_OPAQUE_PYOBJECT.
+// See end of 'extension.c'
+
+
+#undef _Py_OPAQUE_PYOBJECT
+#undef Py_LIMITED_API
+#include "Python.h"
+
+
+// (repeated definition to avoid creating a header)
+extern PyObject *testcext_create_moduledef(
+    const char *name, const char *doc,
+    PyMethodDef *methods, PyModuleDef_Slot *slots);
+
+PyObject *testcext_create_moduledef(
+    const char *name, const char *doc,
+    PyMethodDef *methods, PyModuleDef_Slot *slots) {
+
+    static struct PyModuleDef _testcext_module = {
+        PyModuleDef_HEAD_INIT,
+    };
+    if (!_testcext_module.m_name) {
+        _testcext_module.m_name = name;
+        _testcext_module.m_doc = doc;
+        _testcext_module.m_methods = methods;
+        _testcext_module.m_slots = slots;
+    }
+    return PyModuleDef_Init(&_testcext_module);
+}

Lib/test/test_cext/extension.c

@@ -78,6 +78,9 @@ _testcext_exec(
     return 0;
 }
 
+#define _FUNC_NAME(NAME) PyInit_ ## NAME
+#define FUNC_NAME(NAME) _FUNC_NAME(NAME)
+
 // Converting from function pointer to void* has undefined behavior, but
 // works on all known platforms, and CPython's module and type slots currently
 // need it.
@@ -96,9 +99,10 @@ static PyModuleDef_Slot _testcext_slots[] = {
 
 _Py_COMP_DIAG_POP
 
-
 PyDoc_STRVAR(_testcext_doc, "C test extension.");
 
+#ifndef _Py_OPAQUE_PYOBJECT
+
 static struct PyModuleDef _testcext_module = {
     PyModuleDef_HEAD_INIT,  // m_base
     STR(MODULE_NAME),  // m_name
@@ -112,11 +116,30 @@ static struct PyModuleDef _testcext_module = {
 };
 
 
-#define _FUNC_NAME(NAME) PyInit_ ## NAME
-#define FUNC_NAME(NAME) _FUNC_NAME(NAME)
-
 PyMODINIT_FUNC
 FUNC_NAME(MODULE_NAME)(void)
 {
     return PyModuleDef_Init(&_testcext_module);
 }
+
+#else  // _Py_OPAQUE_PYOBJECT
+
+// Opaque PyObject means that PyModuleDef is also opaque and cannot be
+// declared statically. See PEP 793.
+// So, this part of module creation is split into a separate source file
+// which uses non-limited API.
+
+// (repeated definition to avoid creating a header)
+extern PyObject *testcext_create_moduledef(
+    const char *name, const char *doc,
+    PyMethodDef *methods, PyModuleDef_Slot *slots);
+
+
+PyMODINIT_FUNC
+FUNC_NAME(MODULE_NAME)(void)
+{
+    return testcext_create_moduledef(
+        STR(MODULE_NAME), _testcext_doc, _testcext_methods, _testcext_slots);
+}
+
+#endif  // _Py_OPAQUE_PYOBJECT

Lib/test/test_cext/setup.py

@@ -59,8 +59,11 @@ def main():
     std = os.environ.get("CPYTHON_TEST_STD", "")
     module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
     limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
+    opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", ""))
     internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0")))
 
+    sources = [SOURCE]
+
     if not internal:
         cflags = list(PUBLIC_CFLAGS)
     else:
@@ -93,6 +96,11 @@ def main():
         version = sys.hexversion
         cflags.append(f'-DPy_LIMITED_API={version:#x}')
 
+    # Define _Py_OPAQUE_PYOBJECT macro
+    if opaque_pyobject:
+        cflags.append(f'-D_Py_OPAQUE_PYOBJECT')
+        sources.append('create_moduledef.c')
+
     if internal:
         cflags.append('-DTEST_INTERNAL_C_API=1')
 
@@ -120,7 +128,7 @@ def main():
 
     ext = Extension(
         module_name,
-        sources=[SOURCE],
+        sources=sources,
         extra_compile_args=cflags,
         include_dirs=include_dirs,
         library_dirs=library_dirs)