gh-91271: Document which parts of structs are in limited API/stable ABI (GH-32196) (GH-95711)

Co-authored-by: Erlend Egeberg Aasland <erlend.aasland@innova.no>

Doc/tools/extensions/c_annotations.py

@@ -36,6 +36,7 @@
     'type': 'type',
     'macro': 'macro',
     'type': 'type',
+    'member': 'member',
 }
 
 
@@ -100,6 +101,12 @@ def add_annotations(self, app, doctree):
             # Stable ABI annotation. These have two forms:
             #   Part of the [Stable ABI](link).
             #   Part of the [Stable ABI](link) since version X.Y.
+            # For structs, there's some more info in the message:
+            #   Part of the [Limited API](link) (as an opaque struct).
+            #   Part of the [Stable ABI](link) (including all members).
+            #   Part of the [Limited API](link) (Only some members are part
+            #       of the stable ABI.).
+            # ... all of which can have "since version X.Y" appended.
             record = self.stable_abi_data.get(name)
             if record:
                 if record['role'] != objtype:
@@ -113,15 +120,27 @@ def add_annotations(self, app, doctree):
                 ref_node = addnodes.pending_xref(
                     'Stable ABI', refdomain="std", reftarget='stable',
                     reftype='ref', refexplicit="False")
+                struct_abi_kind = record['struct_abi_kind']
+                if struct_abi_kind in {'opaque', 'members'}:
+                    ref_node += nodes.Text('Limited API')
+                else:
                     ref_node += nodes.Text('Stable ABI')
                 emph_node += ref_node
+                if struct_abi_kind == 'opaque':
+                    emph_node += nodes.Text(' (as an opaque struct)')
+                elif struct_abi_kind == 'full-abi':
+                    emph_node += nodes.Text(' (including all members)')
                 if record['ifdef_note']:
                     emph_node += nodes.Text(' ' + record['ifdef_note'])
                 if stable_added == '3.2':
                     # Stable ABI was introduced in 3.2.
-                    emph_node += nodes.Text('.')
+                    pass
                 else:
-                    emph_node += nodes.Text(f' since version {stable_added}.')
+                    emph_node += nodes.Text(f' since version {stable_added}')
+                emph_node += nodes.Text('.')
+                if struct_abi_kind == 'members':
+                    emph_node += nodes.Text(
+                        ' (Only some members are part of the stable ABI.)')
                 node.insert(0, emph_node)
 
             # Return value annotation

Misc/NEWS.d/next/Documentation/2022-03-30-17-08-12.bpo-47115.R3wt3i.rst

@@ -0,0 +1,2 @@
+The documentation now lists which members of C structs are part of the
+:ref:`Limited API/Stable ABI <stable>`.

Misc/stable_abi.txt

@@ -10,44 +10,111 @@
 # and PC/pythonXYstub.def
 
 
+# The current format is a simple line-based one with significant indentation.
+# Anything after a hash is a comment.
+
+# There are these kinds of top-level "items":
+# - struct: A C struct. Currently this file does not distinguish between:
+#   - opaque structs, which the Limited API only handles via pointers
+#     (so these can change at any time)
+#   - structs where only certain members are part of the stable ABI (e.g.
+#     PyObject)
+#   - structs which must not be changed at all (e.g. PyType_Slot, which is
+#     fully defined and used in arrays)
+# - function: A function that must be kept available (and exported, i.e. not
+#   converted to a macro).
+# - const: A simple value, defined with `#define`.
+# - macro: A preprocessor macro more complex than a simple `const` value.
+# - data: An exported object, which must continue to be available but its exact
+#   value may change.
+# - typedef: A C typedef which is used in other definitions in the limited API.
+#   Its size/layout/signature must not change.
+
+# Each top-level item can have details defined below it:
+# - added: The version in which the item was added to the stable ABI.
+# - ifdef: A feature macro: the item is only available if this macro is defined
+# - abi_only: If present, the item is not part of the Limited API, but it *is*
+#   part of the stable ABI. The item will not show up in user-facing docs.
+#   Typically used for:
+#   - private functions called by public macros, e.g. _Py_BuildValue_SizeT
+#   - items that were part of the limited API in the past, and must remain part
+#     of the stable ABI.
+#   - a combination of the above (functions that were called by macros that
+#     were public in the past)
+
+# For structs, one of the following must be set:
+# - opaque: The struct name is available in the Limited API, but its members
+#   are not. Users must manipulate it via pointers.
+# - members: Space-separated list of members which are part of the
+#   Limited API and Stable ABI.
+#   Members that aren't listed are not accessible to applications.
+# - full-abi: The entire struct -- all its members and its size -- is part of
+#   the Stable ABI, and must not change.
+
+# Removing items from this file is generally not allowed, and additions should
+# be considered with that in mind. See the devguide for exact rules:
+#    https://devguide.python.org/c-api/#limited-api
+
+# User-facing docs are at:
+#    https://docs.python.org/3/c-api/stable.html#stable
+
+
 # Mentioned in PEP 384:
 
 struct PyObject
     added 3.2
+    members ob_refcnt ob_type
 struct PyVarObject
     added 3.2
+    members ob_base ob_size
 struct PyMethodDef
     added 3.2
+    full-abi
 struct PyMemberDef
     added 3.2
+    full-abi
 struct PyGetSetDef
     added 3.2
+    full-abi
 struct PyModuleDef_Base
     added 3.2
+    full-abi
 struct PyModuleDef
     added 3.2
+    full-abi
 struct PyStructSequence_Field
     added 3.2
+    full-abi
 struct PyStructSequence_Desc
     added 3.2
+    full-abi
 struct PyType_Slot
     added 3.2
+    full-abi
 struct PyType_Spec
     added 3.2
+    full-abi
 struct PyThreadState
     added 3.2
+    opaque
 struct PyInterpreterState
     added 3.2
+    opaque
 struct PyFrameObject
     added 3.2
+    opaque
 struct symtable
     added 3.2
+    opaque
 struct PyWeakReference
     added 3.2
+    opaque
 struct PyLongObject
     added 3.2
+    opaque
 struct PyTypeObject
     added 3.2
+    opaque
 
 function PyType_FromSpec
     added 3.2
@@ -259,11 +326,11 @@ typedef newfunc
     added 3.2
 typedef allocfunc
     added 3.2
-struct PyCFunction
+typedef PyCFunction
     added 3.2
-struct PyCFunctionWithKeywords
+typedef PyCFunctionWithKeywords
     added 3.2
-struct PyCapsule_Destructor
+typedef PyCapsule_Destructor
     added 3.2
 typedef getter
     added 3.2

Tools/scripts/stable_abi.py

@@ -119,6 +119,8 @@ class ABIItem:
     contents: list = dataclasses.field(default_factory=list)
     abi_only: bool = False
     ifdef: str = None
+    struct_abi_kind: str = None
+    members: list = None
 
     KINDS = frozenset({
         'struct', 'function', 'macro', 'data', 'const', 'typedef',
@@ -173,6 +175,15 @@ def raise_error(msg):
             if parent.kind not in {'function', 'data'}:
                 raise_error(f'{kind} cannot go in {parent.kind}')
             parent.abi_only = True
+        elif kind in {'members', 'full-abi', 'opaque'}:
+            if parent.kind not in {'struct'}:
+                raise_error(f'{kind} cannot go in {parent.kind}')
+            if prev := getattr(parent, 'struct_abi_kind', None):
+                raise_error(
+                    f'{parent.name} already has {prev}, cannot add {kind}')
+            parent.struct_abi_kind = kind
+            if kind == 'members':
+                parent.members = content.split()
         else:
             raise_error(f"unknown kind {kind!r}")
         levels.append((entry, level))
@@ -246,7 +257,9 @@ def sort_key(item):
 def gen_doc_annotations(manifest, args, outfile):
     """Generate/check the stable ABI list for documentation annotations"""
     writer = csv.DictWriter(
-        outfile, ['role', 'name', 'added', 'ifdef_note'], lineterminator='\n')
+        outfile,
+        ['role', 'name', 'added', 'ifdef_note', 'struct_abi_kind'],
+        lineterminator='\n')
     writer.writeheader()
     for item in manifest.select(REST_ROLES.keys(), include_abi_only=False):
         if item.ifdef:
@@ -257,7 +270,13 @@ def gen_doc_annotations(manifest, args, outfile):
             'role': REST_ROLES[item.kind],
             'name': item.name,
             'added': item.added,
-            'ifdef_note': ifdef_note})
+            'ifdef_note': ifdef_note,
+            'struct_abi_kind': item.struct_abi_kind})
+        for member_name in item.members or ():
+            writer.writerow({
+                'role': 'member',
+                'name': f'{item.name}.{member_name}',
+                'added': item.added})
 
 def generate_or_check(manifest, args, path, func):
     """Generate/check a file with a single generator