bpo-47115: Document which parts of structs are in limited API/stable ABI (GH-32196)

Co-authored-by: Erlend Egeberg Aasland <erlend.aasland@innova.no>

Doc/c-api/frame.rst

@@ -7,10 +7,12 @@ Frame Objects
 
    The C structure of the objects used to describe frame objects.
 
-   The structure is not part of the C API.
+   There are no public members in this structure.
 
    .. versionchanged:: 3.11
-      The structure moved to the internal C API headers.
+      The members of this structure were removed from the public C API.
+      Refer to the :ref:`What's New entry <pyframeobject-3.11-hiding>`
+      for details.
 
 The :c:func:`PyEval_GetFrame` and :c:func:`PyThreadState_GetFrame` functions
 can be used to get a frame object.

Doc/tools/extensions/c_annotations.py

@@ -36,6 +36,7 @@
     'type': 'type',
     'macro': 'macro',
     'type': 'type',
+    'member': 'member',
 }
 
 
@@ -100,6 +101,12 @@ def add_annotations(self, app, doctree):
             # Stable ABI annotation. These have two forms:
             #   Part of the [Stable ABI](link).
             #   Part of the [Stable ABI](link) since version X.Y.
+            # For structs, there's some more info in the message:
+            #   Part of the [Limited API](link) (as an opaque struct).
+            #   Part of the [Stable ABI](link) (including all members).
+            #   Part of the [Limited API](link) (Only some members are part
+            #       of the stable ABI.).
+            # ... all of which can have "since version X.Y" appended.
             record = self.stable_abi_data.get(name)
             if record:
                 if record['role'] != objtype:
@@ -113,15 +120,27 @@ def add_annotations(self, app, doctree):
                 ref_node = addnodes.pending_xref(
                     'Stable ABI', refdomain="std", reftarget='stable',
                     reftype='ref', refexplicit="False")
+                struct_abi_kind = record['struct_abi_kind']
+                if struct_abi_kind in {'opaque', 'members'}:
+                    ref_node += nodes.Text('Limited API')
+                else:
                     ref_node += nodes.Text('Stable ABI')
                 emph_node += ref_node
+                if struct_abi_kind == 'opaque':
+                    emph_node += nodes.Text(' (as an opaque struct)')
+                elif struct_abi_kind == 'full-abi':
+                    emph_node += nodes.Text(' (including all members)')
                 if record['ifdef_note']:
                     emph_node += nodes.Text(' ' + record['ifdef_note'])
                 if stable_added == '3.2':
                     # Stable ABI was introduced in 3.2.
-                    emph_node += nodes.Text('.')
+                    pass
                 else:
-                    emph_node += nodes.Text(f' since version {stable_added}.')
+                    emph_node += nodes.Text(f' since version {stable_added}')
+                emph_node += nodes.Text('.')
+                if struct_abi_kind == 'members':
+                    emph_node += nodes.Text(
+                        ' (Only some members are part of the stable ABI.)')
                 node.insert(0, emph_node)
 
             # Return value annotation

Doc/whatsnew/3.11.rst

@@ -1214,8 +1214,10 @@ Porting to Python 3.11
   which are not available in the limited C API.
   (Contributed by Victor Stinner in :issue:`46007`.)
 
-* The :c:type:`PyFrameObject` structure member has been moved to the internal C
+.. _pyframeobject-3.11-hiding:
-  API headers.
+
+* The :c:type:`PyFrameObject` structure members have been removed from the
+  public C API.
 
   While the documentation notes that the :c:type:`PyFrameObject` fields are
   subject to change at any time, they have been stable for a long time and were

Misc/NEWS.d/next/Documentation/2022-03-30-17-08-12.bpo-47115.R3wt3i.rst

@@ -0,0 +1,2 @@
+The documentation now lists which members of C structs are part of the
+:ref:`Limited API/Stable ABI <stable>`.

Misc/stable_abi.txt

@@ -42,6 +42,15 @@
 #   - a combination of the above (functions that were called by macros that
 #     were public in the past)
 
+# For structs, one of the following must be set:
+# - opaque: The struct name is available in the Limited API, but its members
+#   are not. Users must manipulate it via pointers.
+# - members: Space-separated list of members which are part of the
+#   Limited API and Stable ABI.
+#   Members that aren't listed are not accessible to applications.
+# - full-abi: The entire struct -- all its members and its size -- is part of
+#   the Stable ABI, and must not change.
+
 # Removing items from this file is generally not allowed, and additions should
 # be considered with that in mind. See the devguide for exact rules:
 #    https://devguide.python.org/c-api/#limited-api
@@ -54,40 +63,58 @@
 
 struct PyObject
     added 3.2
+    members ob_refcnt ob_type
 struct PyVarObject
     added 3.2
+    members ob_base ob_size
 struct PyMethodDef
     added 3.2
+    full-abi
 struct PyMemberDef
     added 3.2
+    full-abi
 struct PyGetSetDef
     added 3.2
+    full-abi
 struct PyModuleDef_Base
     added 3.2
+    full-abi
 struct PyModuleDef
     added 3.2
+    full-abi
 struct PyStructSequence_Field
     added 3.2
+    full-abi
 struct PyStructSequence_Desc
     added 3.2
+    full-abi
 struct PyType_Slot
     added 3.2
+    full-abi
 struct PyType_Spec
     added 3.2
+    full-abi
 struct PyThreadState
     added 3.2
+    opaque
 struct PyInterpreterState
     added 3.2
+    opaque
 struct PyFrameObject
     added 3.2
+    opaque
 struct symtable
     added 3.2
+    opaque
 struct PyWeakReference
     added 3.2
+    opaque
 struct PyLongObject
     added 3.2
+    opaque
 struct PyTypeObject
     added 3.2
+    opaque
 
 function PyType_FromSpec
     added 3.2
@@ -299,11 +326,11 @@ typedef newfunc
     added 3.2
 typedef allocfunc
     added 3.2
-struct PyCFunction
+typedef PyCFunction
     added 3.2
-struct PyCFunctionWithKeywords
+typedef PyCFunctionWithKeywords
     added 3.2
-struct PyCapsule_Destructor
+typedef PyCapsule_Destructor
     added 3.2
 typedef getter
     added 3.2
@@ -2194,6 +2221,7 @@ data PyStructSequence_UnnamedField
 # Add stable Py_buffer API in Python 3.11 (https://bugs.python.org/issue45459)
 struct Py_buffer
     added 3.11
+    full-abi
 function PyObject_CheckBuffer
     added 3.11
 function PyObject_GetBuffer

Tools/scripts/stable_abi.py

@@ -118,6 +118,8 @@ class ABIItem:
     contents: list = dataclasses.field(default_factory=list)
     abi_only: bool = False
     ifdef: str = None
+    struct_abi_kind: str = None
+    members: list = None
 
     KINDS = frozenset({
         'struct', 'function', 'macro', 'data', 'const', 'typedef',
@@ -172,6 +174,15 @@ def raise_error(msg):
             if parent.kind not in {'function', 'data'}:
                 raise_error(f'{kind} cannot go in {parent.kind}')
             parent.abi_only = True
+        elif kind in {'members', 'full-abi', 'opaque'}:
+            if parent.kind not in {'struct'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            if prev := getattr(parent, 'struct_abi_kind', None):
+                raise_error(
+                    f'{parent.name} already has {prev}, cannot add {kind}')
+            parent.struct_abi_kind = kind
+            if kind == 'members':
+                parent.members = content.split()
         else:
             raise_error(f"unknown kind {kind!r}")
             # When adding more, update the comment in stable_abi.txt.
@@ -246,7 +257,9 @@ def sort_key(item):
 def gen_doc_annotations(manifest, args, outfile):
     """Generate/check the stable ABI list for documentation annotations"""
     writer = csv.DictWriter(
-        outfile, ['role', 'name', 'added', 'ifdef_note'], lineterminator='\n')
+        outfile,
+        ['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'],
+        lineterminator='\n')
     writer.writeheader()
     for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
         if item.ifdef:
@@ -257,7 +270,13 @@ def gen_doc_annotations(manifest, args, outfile):
             'role': REST_ROLES[item.kind],
             'name': item.name,
             'added': item.added,
-            'ifdef_note': ifdef_note})
+            'ifdef_note': ifdef_note,
+            'struct_abi_kind': item.struct_abi_kind})
+        for member_name in item.members or ():
+            writer.writerow({
+                'role': 'member',
+                'name': f'{item.name}.{member_name}',
+                'added': item.added})
 
 @generator("ctypes_test", 'Lib/test/test_stable_abi_ctypes.py')
 def gen_ctypes_test(manifest, args, outfile):