Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-12-08 06:10:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

[3.14] gh-133403: Check generate_stdlib_module_names and check_extension_modules with mypy (GH-137546) (#137679)

Browse source 
Co-authored-by: sobolevn <mail@sobolevn.me>
This commit is contained in:
Miss Islington (bot) 2025-08-13 12:42:26 +02:00 • committed by GitHub
parent c60289c7f2
commit b7df473260
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 65 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/mypy.yml vendored
View file

@ -13,9 +13,11 @@ on:
- "Lib/test/libregrtest/**"
- "Lib/tomllib/**"
- "Misc/mypy/**"
- "Tools/build/check_extension_modules.py"
- "Tools/build/compute-changes.py"
- "Tools/build/deepfreeze.py"
- "Tools/build/generate_sbom.py"
- "Tools/build/generate_stdlib_module_names.py"
- "Tools/build/generate-build-details.py"
- "Tools/build/verify_ensurepip_wheels.py"
- "Tools/build/update_file.py"

80
Tools/build/check_extension_modules.py
View file

@ -17,9 +17,11 @@
See --help for more information
"""
from __future__ import annotations
import _imp
import argparse
import collections
import enum
import logging
import os
@ -29,13 +31,16 @@
import sysconfig
import warnings
from collections.abc import Iterable
from importlib._bootstrap import _load as bootstrap_load
from importlib._bootstrap import ( # type: ignore[attr-defined]
_load as bootstrap_load,
)
from importlib.machinery import (
BuiltinImporter,
ExtensionFileLoader,
ModuleSpec,
)
from importlib.util import spec_from_file_location, spec_from_loader
from typing import NamedTuple
SRC_DIR = pathlib.Path(__file__).parent.parent.parent
@ -112,6 +117,7 @@
)
@enum.unique
class ModuleState(enum.Enum):
# Makefile state "yes"
BUILTIN = "builtin"
@ -123,11 +129,13 @@ class ModuleState(enum.Enum):
# disabled by Setup / makesetup rule
DISABLED_SETUP = "disabled_setup"
def __bool__(self):
def __bool__(self) -> bool:
return self.value in {"builtin", "shared"}
ModuleInfo = collections.namedtuple("ModuleInfo", "name state")
class ModuleInfo(NamedTuple):
name: str
state: ModuleState
class ModuleChecker:
@ -135,9 +143,9 @@ class ModuleChecker:
setup_files = (
# see end of configure.ac
"Modules/Setup.local",
"Modules/Setup.stdlib",
"Modules/Setup.bootstrap",
pathlib.Path("Modules/Setup.local"),
pathlib.Path("Modules/Setup.stdlib"),
pathlib.Path("Modules/Setup.bootstrap"),
SRC_DIR / "Modules/Setup",
)
@ -149,15 +157,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False):
self.builddir = self.get_builddir()
self.modules = self.get_modules()
self.builtin_ok = []
self.shared_ok = []
self.failed_on_import = []
self.missing = []
self.disabled_configure = []
self.disabled_setup = []
self.notavailable = []
self.builtin_ok: list[ModuleInfo] = []
self.shared_ok: list[ModuleInfo] = []
self.failed_on_import: list[ModuleInfo] = []
self.missing: list[ModuleInfo] = []
self.disabled_configure: list[ModuleInfo] = []
self.disabled_setup: list[ModuleInfo] = []
self.notavailable: list[ModuleInfo] = []
def check(self):
def check(self) -> None:
if not hasattr(_imp, 'create_dynamic'):
logger.warning(
('Dynamic extensions not supported '
@ -189,10 +197,10 @@ def check(self):
assert modinfo.state == ModuleState.SHARED
self.shared_ok.append(modinfo)
def summary(self, *, verbose: bool = False):
def summary(self, *, verbose: bool = False) -> None:
longest = max([len(e.name) for e in self.modules], default=0)
def print_three_column(modinfos: list[ModuleInfo]):
def print_three_column(modinfos: list[ModuleInfo]) -> None:
names = [modinfo.name for modinfo in modinfos]
names.sort(key=str.lower)
# guarantee zip() doesn't drop anything
@ -262,12 +270,12 @@ def print_three_column(modinfos: list[ModuleInfo]):
f"{len(self.failed_on_import)} failed on import)"
)
def check_strict_build(self):
def check_strict_build(self) -> None:
"""Fail if modules are missing and it's a strict build"""
if self.strict_extensions_build and (self.failed_on_import or self.missing):
raise RuntimeError("Failed to build some stdlib modules")
def list_module_names(self, *, all: bool = False) -> set:
def list_module_names(self, *, all: bool = False) -> set[str]:
names = {modinfo.name for modinfo in self.modules}
if all:
names.update(WINDOWS_MODULES)
@ -280,9 +288,9 @@ def get_builddir(self) -> pathlib.Path:
except FileNotFoundError:
logger.error("%s must be run from the top build directory", __file__)
raise
builddir = pathlib.Path(builddir)
logger.debug("%s: %s", self.pybuilddir_txt, builddir)
return builddir
builddir_path = pathlib.Path(builddir)
logger.debug("%s: %s", self.pybuilddir_txt, builddir_path)
return builddir_path
def get_modules(self) -> list[ModuleInfo]:
"""Get module info from sysconfig and Modules/Setup* files"""
@ -367,7 +375,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
case ["*disabled*"]:
state = ModuleState.DISABLED
case ["*noconfig*"]:
state = None
continue
case [*items]:
if state == ModuleState.DISABLED:
# *disabled* can disable multiple modules per line
@ -384,26 +392,33 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
"""Get ModuleSpec for builtin or extension module"""
if modinfo.state == ModuleState.SHARED:
location = os.fspath(self.get_location(modinfo))
mod_location = self.get_location(modinfo)
assert mod_location is not None
location = os.fspath(mod_location)
loader = ExtensionFileLoader(modinfo.name, location)
return spec_from_file_location(modinfo.name, location, loader=loader)
spec = spec_from_file_location(modinfo.name, location, loader=loader)
assert spec is not None
return spec
elif modinfo.state == ModuleState.BUILTIN:
return spec_from_loader(modinfo.name, loader=BuiltinImporter)
spec = spec_from_loader(modinfo.name, loader=BuiltinImporter)
assert spec is not None
return spec
else:
raise ValueError(modinfo)
def get_location(self, modinfo: ModuleInfo) -> pathlib.Path:
def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None:
"""Get shared library location in build directory"""
if modinfo.state == ModuleState.SHARED:
return self.builddir / f"{modinfo.name}{self.ext_suffix}"
else:
return None
def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec):
def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None:
"""Check that the module file is present and not empty"""
if spec.loader is BuiltinImporter:
if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap]
return
try:
assert spec.origin is not None
st = os.stat(spec.origin)
except FileNotFoundError:
logger.error("%s (%s) is missing", modinfo.name, spec.origin)
@ -411,7 +426,7 @@ def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec):
if not st.st_size:
raise ImportError(f"{spec.origin} is an empty file")
def check_module_import(self, modinfo: ModuleInfo):
def check_module_import(self, modinfo: ModuleInfo) -> None:
"""Attempt to import module and report errors"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
@ -430,7 +445,7 @@ def check_module_import(self, modinfo: ModuleInfo):
logger.exception("Importing extension '%s' failed!", modinfo.name)
raise
def check_module_cross(self, modinfo: ModuleInfo):
def check_module_cross(self, modinfo: ModuleInfo) -> None:
"""Sanity check for cross compiling"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
@ -443,6 +458,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None:
failed_name = f"{modinfo.name}_failed{self.ext_suffix}"
builddir_path = self.get_location(modinfo)
assert builddir_path is not None
if builddir_path.is_symlink():
symlink = builddir_path
module_path = builddir_path.resolve().relative_to(os.getcwd())
@ -466,7 +482,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None:
logger.debug("Rename '%s' -> '%s'", module_path, failed_path)
def main():
def main() -> None:
args = parser.parse_args()
if args.debug:
args.verbose = True

23
Tools/build/generate_stdlib_module_names.py
View file

@ -1,9 +1,12 @@
# This script lists the names of standard library modules
# to update Python/stdlib_module_names.h
from __future__ import annotations
import _imp
import os.path
import sys
import sysconfig
from typing import TextIO
from check_extension_modules import ModuleChecker
@ -48,12 +51,12 @@
}
# Built-in modules
def list_builtin_modules(names):
def list_builtin_modules(names: set[str]) -> None:
names |= set(sys.builtin_module_names)
# Pure Python modules (Lib/*.py)
def list_python_modules(names):
def list_python_modules(names: set[str]) -> None:
for filename in os.listdir(STDLIB_PATH):
if not filename.endswith(".py"):
continue
@ -62,7 +65,7 @@ def list_python_modules(names):
# Packages in Lib/
def list_packages(names):
def list_packages(names: set[str]) -> None:
for name in os.listdir(STDLIB_PATH):
if name in IGNORE:
continue
@ -76,16 +79,16 @@ def list_packages(names):
# Built-in and extension modules built by Modules/Setup*
# includes Windows and macOS extensions.
def list_modules_setup_extensions(names):
def list_modules_setup_extensions(names: set[str]) -> None:
checker = ModuleChecker()
names.update(checker.list_module_names(all=True))
# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
# Use the "./Programs/_testembed list_frozen" command.
def list_frozen(names):
def list_frozen(names: set[str]) -> None:
submodules = set()
for name in _imp._frozen_module_names():
for name in _imp._frozen_module_names(): # type: ignore[attr-defined]
# To skip __hello__, __hello_alias__ and etc.
if name.startswith('__'):
continue
@ -101,8 +104,8 @@ def list_frozen(names):
raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
def list_modules():
names = set()
def list_modules() -> set[str]:
names: set[str] = set()
list_builtin_modules(names)
list_modules_setup_extensions(names)
@ -127,7 +130,7 @@ def list_modules():
return names
def write_modules(fp, names):
def write_modules(fp: TextIO, names: set[str]) -> None:
print(f"// Auto-generated by {SCRIPT_NAME}.",
file=fp)
print("// List used to create sys.stdlib_module_names.", file=fp)
@ -138,7 +141,7 @@ def write_modules(fp, names):
print("};", file=fp)
def main():
def main() -> None:
if not sysconfig.is_python_build():
print(f"ERROR: {sys.executable} is not a Python build",
file=sys.stderr)

2
Tools/build/mypy.ini
View file

@ -3,10 +3,12 @@
# Please, when adding new files here, also add them to:
# .github/workflows/mypy.yml
files =
Tools/build/check_extension_modules.py,
Tools/build/compute-changes.py,
Tools/build/deepfreeze.py,
Tools/build/generate-build-details.py,
Tools/build/generate_sbom.py,
Tools/build/generate_stdlib_module_names.py,
Tools/build/verify_ensurepip_wheels.py,
Tools/build/update_file.py,
Tools/build/umarshal.py