bpo-42545: Check that all symbols in the limited ABI are exported (GH-23616)

Tools/scripts/stable_abi.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python
+
+import argparse
+import glob
+import re
+import pathlib
+import subprocess
+import sys
+import sysconfig
+
+EXCLUDED_HEADERS = {
+    "bytes_methods.h",
+    "cellobject.h",
+    "classobject.h",
+    "code.h",
+    "compile.h",
+    "datetime.h",
+    "dtoa.h",
+    "frameobject.h",
+    "funcobject.h",
+    "genobject.h",
+    "longintrepr.h",
+    "parsetok.h",
+    "pyarena.h",
+    "pyatomic.h",
+    "pyctype.h",
+    "pydebug.h",
+    "pytime.h",
+    "symtable.h",
+    "token.h",
+    "ucnhash.h",
+}
+
+
+def get_exported_symbols(library, dynamic=False):
+    # Only look at dynamic symbols
+    args = ["nm", "--no-sort"]
+    if dynamic:
+        args.append("--dynamic")
+    args.append(library)
+    proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True)
+    if proc.returncode:
+        sys.stdout.write(proc.stdout)
+        sys.exit(proc.returncode)
+
+    stdout = proc.stdout.rstrip()
+    if not stdout:
+        raise Exception("command output is empty")
+
+    for line in stdout.splitlines():
+        # Split line '0000000000001b80 D PyTextIOWrapper_Type'
+        if not line:
+            continue
+
+        parts = line.split(maxsplit=2)
+        if len(parts) < 3:
+            continue
+
+        symbol = parts[-1]
+        yield symbol
+
+
+def check_library(library, abi_funcs, dynamic=False):
+    available_symbols = set(get_exported_symbols(library, dynamic))
+    missing_symbols = abi_funcs - available_symbols
+    if missing_symbols:
+        print(
+            f"Some symbols from the stable ABI are missing: {', '.join(missing_symbols)}"
+        )
+        return 1
+    return 0
+
+
+def generate_limited_api_symbols(args):
+    if hasattr(sys, "gettotalrefcount"):
+        print(
+            "Stable ABI symbols cannot be generated from a debug build", file=sys.stderr
+        )
+        sys.exit(1)
+    library = sysconfig.get_config_var("LIBRARY")
+    ldlibrary = sysconfig.get_config_var("LDLIBRARY")
+    if ldlibrary != library:
+        raise Exception("Limited ABI symbols can only be generated from a static build")
+    available_symbols = {
+        symbol for symbol in get_exported_symbols(library) if symbol.startswith("Py")
+    }
+
+    headers = [
+        file
+        for file in pathlib.Path("Include").glob("*.h")
+        if file.name not in EXCLUDED_HEADERS
+    ]
+    stable_data, stable_exported_data, stable_functions = get_limited_api_definitions(
+        headers
+    )
+    macros = get_limited_api_macros(headers)
+
+    stable_symbols = {
+        symbol
+        for symbol in (stable_functions | stable_exported_data | stable_data | macros)
+        if symbol.startswith("Py") and symbol in available_symbols
+    }
+    with open(args.output_file, "w") as output_file:
+        output_file.write(f"# File generated by 'make regen-limited-abi'\n")
+        output_file.write(
+            f"# This is NOT an authoritative list of stable ABI symbols\n"
+        )
+        for symbol in sorted(stable_symbols):
+            output_file.write(f"{symbol}\n")
+    sys.exit(0)
+
+
+def get_limited_api_macros(headers):
+    """Run the preprocesor over all the header files in "Include" setting
+    "-DPy_LIMITED_API" to the correct value for the running version of the interpreter
+    and extracting all macro definitions (via adding -dM to the compiler arguments).
+    """
+
+    preprocesor_output_with_macros = subprocess.check_output(
+        sysconfig.get_config_var("CC").split()
+        + [
+            # Prevent the expansion of the exported macros so we can capture them later
+            "-DSIZEOF_WCHAR_T=4",  # The actual value is not important
+            f"-DPy_LIMITED_API={sys.version_info.major << 24 | sys.version_info.minor << 16}",
+            "-I.",
+            "-I./Include",
+            "-dM",
+            "-E",
+        ]
+        + [str(file) for file in headers],
+        text=True,
+        stderr=subprocess.DEVNULL,
+    )
+
+    return {
+        target
+        for _, target in re.findall(
+            r"#define (\w+)\s*(?:\(.*?\))?\s+(\w+)", preprocesor_output_with_macros
+        )
+    }
+
+
+def get_limited_api_definitions(headers):
+    """Run the preprocesor over all the header files in "Include" setting
+    "-DPy_LIMITED_API" to the correct value for the running version of the interpreter.
+
+    The limited API symbols will be extracted from the output of this command as it includes
+    the prototypes and definitions of all the exported symbols that are in the limited api.
+
+    This function does *NOT* extract the macros defined on the limited API
+    """
+    preprocesor_output = subprocess.check_output(
+        sysconfig.get_config_var("CC").split()
+        + [
+            # Prevent the expansion of the exported macros so we can capture them later
+            "-DPyAPI_FUNC=__PyAPI_FUNC",
+            "-DPyAPI_DATA=__PyAPI_DATA",
+            "-DEXPORT_DATA=__EXPORT_DATA",
+            "-D_Py_NO_RETURN=",
+            "-DSIZEOF_WCHAR_T=4",  # The actual value is not important
+            f"-DPy_LIMITED_API={sys.version_info.major << 24 | sys.version_info.minor << 16}",
+            "-I.",
+            "-I./Include",
+            "-E",
+        ]
+        + [str(file) for file in headers],
+        text=True,
+        stderr=subprocess.DEVNULL,
+    )
+    stable_functions = set(
+        re.findall(r"__PyAPI_FUNC\(.*?\)\s*(.*?)\s*\(", preprocesor_output)
+    )
+    stable_exported_data = set(
+        re.findall(r"__EXPORT_DATA\((.*?)\)", preprocesor_output)
+    )
+    stable_data = set(
+        re.findall(r"__PyAPI_DATA\(.*?\)\s*\(?(.*?)\)?\s*;", preprocesor_output)
+    )
+    return stable_data, stable_exported_data, stable_functions
+
+
+def check_symbols(parser_args):
+    with open(parser_args.stable_abi_file, "r") as filename:
+        abi_funcs = {
+            symbol
+            for symbol in filename.read().splitlines()
+            if symbol and not symbol.startswith("#")
+        }
+
+    ret = 0
+    # static library
+    LIBRARY = sysconfig.get_config_var("LIBRARY")
+    if not LIBRARY:
+        raise Exception("failed to get LIBRARY variable from sysconfig")
+    ret = check_library(LIBRARY, abi_funcs)
+
+    # dynamic library
+    LDLIBRARY = sysconfig.get_config_var("LDLIBRARY")
+    if not LDLIBRARY:
+        raise Exception("failed to get LDLIBRARY variable from sysconfig")
+    if LDLIBRARY != LIBRARY:
+        ret |= check_library(LDLIBRARY, abi_funcs, dynamic=True)
+
+    sys.exit(ret)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Process some integers.")
+    subparsers = parser.add_subparsers()
+    check_parser = subparsers.add_parser(
+        "check", help="Check the exported symbols against a given ABI file"
+    )
+    check_parser.add_argument(
+        "stable_abi_file", type=str, help="File with the stable abi functions"
+    )
+    check_parser.set_defaults(func=check_symbols)
+    generate_parser = subparsers.add_parser(
+        "generate",
+        help="Generate symbols from the header files and the exported symbols",
+    )
+    generate_parser.add_argument(
+        "output_file", type=str, help="File to dump the symbols to"
+    )
+    generate_parser.set_defaults(func=generate_limited_api_symbols)
+    args = parser.parse_args()
+    if "func" not in args:
+        parser.error("Either 'check' or 'generate' must be used")
+        sys.exit(1)
+
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()