mirror of
https://github.com/python/cpython.git
synced 2025-12-08 06:10:17 +00:00
ca1e86f9d9
This introduces a Wasmtime configuration file to get some CLI options out of the code for easier manipulation. It also allows for easier tweaking after the Makefile is generated. As well, cut back on the flexibility of specifying HOSTRUNNER for simpler code. The flexibility was never used and so it didn't make sense to keep it around.
526 lines
17 KiB
Python
526 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
|
||
import argparse
|
||
import contextlib
|
||
import functools
|
||
import os
|
||
|
||
try:
|
||
from os import process_cpu_count as cpu_count
|
||
except ImportError:
|
||
from os import cpu_count
|
||
import pathlib
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import sysconfig
|
||
import tempfile
|
||
|
||
HERE = pathlib.Path(__file__).parent
|
||
|
||
CHECKOUT = HERE.parent.parent.parent
|
||
assert (CHECKOUT / "configure").is_file(), (
|
||
"Please update the location of the file"
|
||
)
|
||
|
||
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
|
||
# Build platform can also be found via `config.guess`.
|
||
BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
|
||
|
||
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
|
||
LOCAL_SETUP_MARKER = (
|
||
b"# Generated by Tools/wasm/wasi .\n"
|
||
b"# Required to statically build extension modules."
|
||
)
|
||
|
||
WASI_SDK_VERSION = 29
|
||
|
||
WASMTIME_VAR_NAME = "WASMTIME"
|
||
WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
|
||
|
||
|
||
def separator():
|
||
"""Print a separator line across the terminal width."""
|
||
try:
|
||
tput_output = subprocess.check_output(
|
||
["tput", "cols"], encoding="utf-8"
|
||
)
|
||
except subprocess.CalledProcessError:
|
||
terminal_width = 80
|
||
else:
|
||
terminal_width = int(tput_output.strip())
|
||
print("โฏ" * terminal_width)
|
||
|
||
|
||
def log(emoji, message, *, spacing=None):
|
||
"""Print a notification with an emoji.
|
||
|
||
If 'spacing' is None, calculate the spacing based on the number of code points
|
||
in the emoji as terminals "eat" a space when the emoji has multiple code points.
|
||
"""
|
||
if spacing is None:
|
||
spacing = " " if len(emoji) == 1 else " "
|
||
print("".join([emoji, spacing, message]))
|
||
|
||
|
||
def updated_env(updates={}):
|
||
"""Create a new dict representing the environment to use.
|
||
|
||
The changes made to the execution environment are printed out.
|
||
"""
|
||
env_defaults = {}
|
||
# https://reproducible-builds.org/docs/source-date-epoch/
|
||
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
|
||
try:
|
||
epoch = subprocess.check_output(
|
||
git_epoch_cmd, encoding="utf-8"
|
||
).strip()
|
||
env_defaults["SOURCE_DATE_EPOCH"] = epoch
|
||
except subprocess.CalledProcessError:
|
||
pass # Might be building from a tarball.
|
||
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
|
||
environment = env_defaults | os.environ | updates
|
||
|
||
env_diff = {}
|
||
for key, value in environment.items():
|
||
if os.environ.get(key) != value:
|
||
env_diff[key] = value
|
||
|
||
env_vars = (
|
||
f"\n {key}={item}" for key, item in sorted(env_diff.items())
|
||
)
|
||
log("๐", f"Environment changes:{''.join(env_vars)}")
|
||
|
||
return environment
|
||
|
||
|
||
def subdir(working_dir, *, clean_ok=False):
|
||
"""Decorator to change to a working directory."""
|
||
|
||
def decorator(func):
|
||
@functools.wraps(func)
|
||
def wrapper(context):
|
||
nonlocal working_dir
|
||
|
||
if callable(working_dir):
|
||
working_dir = working_dir(context)
|
||
separator()
|
||
log("๐", os.fsdecode(working_dir))
|
||
if (
|
||
clean_ok
|
||
and getattr(context, "clean", False)
|
||
and working_dir.exists()
|
||
):
|
||
log("๐ฎ", "Deleting directory (--clean)...")
|
||
shutil.rmtree(working_dir)
|
||
|
||
working_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
with contextlib.chdir(working_dir):
|
||
return func(context, working_dir)
|
||
|
||
return wrapper
|
||
|
||
return decorator
|
||
|
||
|
||
def call(command, *, context=None, quiet=False, logdir=None, **kwargs):
|
||
"""Execute a command.
|
||
|
||
If 'quiet' is true, then redirect stdout and stderr to a temporary file.
|
||
"""
|
||
if context is not None:
|
||
quiet = context.quiet
|
||
logdir = context.logdir
|
||
elif quiet and logdir is None:
|
||
raise ValueError("When quiet is True, logdir must be specified")
|
||
|
||
log("โฏ", " ".join(map(str, command)), spacing=" ")
|
||
if not quiet:
|
||
stdout = None
|
||
stderr = None
|
||
else:
|
||
stdout = tempfile.NamedTemporaryFile(
|
||
"w",
|
||
encoding="utf-8",
|
||
delete=False,
|
||
dir=logdir,
|
||
prefix="cpython-wasi-",
|
||
suffix=".log",
|
||
)
|
||
stderr = subprocess.STDOUT
|
||
log("๐", f"Logging output to {stdout.name} (--quiet)...")
|
||
|
||
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
|
||
|
||
|
||
def build_python_path():
|
||
"""The path to the build Python binary."""
|
||
binary = BUILD_DIR / "python"
|
||
if not binary.is_file():
|
||
binary = binary.with_suffix(".exe")
|
||
if not binary.is_file():
|
||
raise FileNotFoundError(
|
||
f"Unable to find `python(.exe)` in {BUILD_DIR}"
|
||
)
|
||
|
||
return binary
|
||
|
||
|
||
def build_python_is_pydebug():
|
||
"""Find out if the build Python is a pydebug build."""
|
||
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
|
||
result = subprocess.run(
|
||
[build_python_path(), "-c", test],
|
||
capture_output=True,
|
||
)
|
||
return bool(result.returncode)
|
||
|
||
|
||
@subdir(BUILD_DIR, clean_ok=True)
|
||
def configure_build_python(context, working_dir):
|
||
"""Configure the build/host Python."""
|
||
if LOCAL_SETUP.exists():
|
||
if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
|
||
log("๐", f"{LOCAL_SETUP} exists ...")
|
||
else:
|
||
log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents")
|
||
else:
|
||
log("๐", f"Creating {LOCAL_SETUP} ...")
|
||
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
|
||
|
||
configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
|
||
if context.args:
|
||
configure.extend(context.args)
|
||
|
||
call(configure, context=context)
|
||
|
||
|
||
@subdir(BUILD_DIR)
|
||
def make_build_python(context, working_dir):
|
||
"""Make/build the build Python."""
|
||
call(["make", "--jobs", str(cpu_count()), "all"], context=context)
|
||
|
||
binary = build_python_path()
|
||
cmd = [
|
||
binary,
|
||
"-c",
|
||
"import sys; "
|
||
"print(f'{sys.version_info.major}.{sys.version_info.minor}')",
|
||
]
|
||
version = subprocess.check_output(cmd, encoding="utf-8").strip()
|
||
|
||
log("๐", f"{binary} {version}")
|
||
|
||
|
||
def find_wasi_sdk():
|
||
"""Find the path to the WASI SDK."""
|
||
wasi_sdk_path = None
|
||
|
||
if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"):
|
||
wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var)
|
||
else:
|
||
opt_path = pathlib.Path("/opt")
|
||
# WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team
|
||
# has said they don't plan to ever do a point release and all of their Git tags
|
||
# lack the ``.0`` suffix.
|
||
# Starting with WASI SDK 23, the tarballs went from containing a directory named
|
||
# ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
|
||
# ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
|
||
potential_sdks = [
|
||
path
|
||
for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*")
|
||
if path.is_dir()
|
||
]
|
||
if len(potential_sdks) == 1:
|
||
wasi_sdk_path = potential_sdks[0]
|
||
elif (default_path := opt_path / "wasi-sdk").is_dir():
|
||
wasi_sdk_path = default_path
|
||
|
||
# Starting with WASI SDK 25, a VERSION file is included in the root
|
||
# of the SDK directory that we can read to warn folks when they are using
|
||
# an unsupported version.
|
||
if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file():
|
||
version_details = version_file.read_text(encoding="utf-8")
|
||
found_version = version_details.splitlines()[0]
|
||
# Make sure there's a trailing dot to avoid false positives if somehow the
|
||
# supported version is a prefix of the found version (e.g. `25` and `2567`).
|
||
if not found_version.startswith(f"{WASI_SDK_VERSION}."):
|
||
major_version = found_version.partition(".")[0]
|
||
log(
|
||
"โ ๏ธ",
|
||
f" Found WASI SDK {major_version}, "
|
||
f"but WASI SDK {WASI_SDK_VERSION} is the supported version",
|
||
)
|
||
|
||
return wasi_sdk_path
|
||
|
||
|
||
def wasi_sdk_env(context):
|
||
"""Calculate environment variables for building with wasi-sdk."""
|
||
wasi_sdk_path = context.wasi_sdk_path
|
||
sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
|
||
env = {
|
||
"CC": "clang",
|
||
"CPP": "clang-cpp",
|
||
"CXX": "clang++",
|
||
"AR": "llvm-ar",
|
||
"RANLIB": "ranlib",
|
||
}
|
||
|
||
for env_var, binary_name in list(env.items()):
|
||
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
|
||
|
||
if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"):
|
||
for compiler in ["CC", "CPP", "CXX"]:
|
||
env[compiler] += f" --sysroot={sysroot}"
|
||
|
||
env["PKG_CONFIG_PATH"] = ""
|
||
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
|
||
map(
|
||
os.fsdecode,
|
||
[sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
|
||
)
|
||
)
|
||
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
|
||
|
||
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
|
||
env["WASI_SYSROOT"] = os.fsdecode(sysroot)
|
||
|
||
env["PATH"] = os.pathsep.join([
|
||
os.fsdecode(wasi_sdk_path / "bin"),
|
||
os.environ["PATH"],
|
||
])
|
||
|
||
return env
|
||
|
||
|
||
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
|
||
def configure_wasi_python(context, working_dir):
|
||
"""Configure the WASI/host build."""
|
||
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
|
||
raise ValueError(
|
||
"WASI-SDK not found; "
|
||
"download from "
|
||
"https://github.com/WebAssembly/wasi-sdk and/or "
|
||
"specify via $WASI_SDK_PATH or --wasi-sdk"
|
||
)
|
||
|
||
config_site = os.fsdecode(HERE / "config.site-wasm32-wasi")
|
||
|
||
wasi_build_dir = working_dir.relative_to(CHECKOUT)
|
||
|
||
python_build_dir = BUILD_DIR / "build"
|
||
lib_dirs = list(python_build_dir.glob("lib.*"))
|
||
assert len(lib_dirs) == 1, (
|
||
f"Expected a single lib.* directory in {python_build_dir}"
|
||
)
|
||
lib_dir = os.fsdecode(lib_dirs[0])
|
||
python_version = lib_dir.rpartition("-")[-1]
|
||
sysconfig_data_dir = (
|
||
f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
|
||
)
|
||
|
||
# Use PYTHONPATH to include sysconfig data which must be anchored to the
|
||
# WASI guest's `/` directory.
|
||
args = {
|
||
"PYTHONPATH": f"/{sysconfig_data_dir}",
|
||
"PYTHON_WASM": working_dir / "python.wasm",
|
||
}
|
||
# Check dynamically for wasmtime in case it was specified manually via
|
||
# `--host-runner`.
|
||
if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
|
||
if wasmtime := shutil.which("wasmtime"):
|
||
args[WASMTIME_VAR_NAME] = wasmtime
|
||
else:
|
||
raise FileNotFoundError(
|
||
"wasmtime not found; download from "
|
||
"https://github.com/bytecodealliance/wasmtime"
|
||
)
|
||
host_runner = context.host_runner.format_map(args)
|
||
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
|
||
build_python = os.fsdecode(build_python_path())
|
||
# The path to `configure` MUST be relative, else `python.wasm` is unable
|
||
# to find the stdlib due to Python not recognizing that it's being
|
||
# executed from within a checkout.
|
||
configure = [
|
||
os.path.relpath(CHECKOUT / "configure", working_dir),
|
||
f"--host={context.host_triple}",
|
||
f"--build={BUILD_DIR.name}",
|
||
f"--with-build-python={build_python}",
|
||
]
|
||
if build_python_is_pydebug():
|
||
configure.append("--with-pydebug")
|
||
if context.args:
|
||
configure.extend(context.args)
|
||
call(
|
||
configure,
|
||
env=updated_env(env_additions | wasi_sdk_env(context)),
|
||
context=context,
|
||
)
|
||
|
||
python_wasm = working_dir / "python.wasm"
|
||
exec_script = working_dir / "python.sh"
|
||
with exec_script.open("w", encoding="utf-8") as file:
|
||
file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
|
||
exec_script.chmod(0o755)
|
||
log("๐", f"Created {exec_script} (--host-runner)... ")
|
||
sys.stdout.flush()
|
||
|
||
|
||
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
|
||
def make_wasi_python(context, working_dir):
|
||
"""Run `make` for the WASI/host build."""
|
||
call(
|
||
["make", "--jobs", str(cpu_count()), "all"],
|
||
env=updated_env(),
|
||
context=context,
|
||
)
|
||
|
||
exec_script = working_dir / "python.sh"
|
||
call([exec_script, "--version"], quiet=False)
|
||
log(
|
||
"๐",
|
||
f"Use `{exec_script.relative_to(context.init_dir)}` "
|
||
"to run CPython w/ the WASI host specified by --host-runner",
|
||
)
|
||
|
||
|
||
def build_all(context):
|
||
"""Build everything."""
|
||
steps = [
|
||
configure_build_python,
|
||
make_build_python,
|
||
configure_wasi_python,
|
||
make_wasi_python,
|
||
]
|
||
for step in steps:
|
||
step(context)
|
||
|
||
|
||
def clean_contents(context):
|
||
"""Delete all files created by this script."""
|
||
if CROSS_BUILD_DIR.exists():
|
||
log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...")
|
||
shutil.rmtree(CROSS_BUILD_DIR)
|
||
|
||
if LOCAL_SETUP.exists():
|
||
if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
|
||
log("๐งน", f"Deleting generated {LOCAL_SETUP} ...")
|
||
|
||
|
||
def main():
|
||
default_host_triple = "wasm32-wasip1"
|
||
default_wasi_sdk = find_wasi_sdk()
|
||
default_host_runner = (
|
||
f"{WASMTIME_HOST_RUNNER_VAR} run "
|
||
# For setting PYTHONPATH to the sysconfig data directory.
|
||
"--env PYTHONPATH={PYTHONPATH} "
|
||
# Map the checkout to / to load the stdlib from /Lib.
|
||
f"--dir {os.fsdecode(CHECKOUT)}::/ "
|
||
# Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept
|
||
# in a config file.
|
||
# We are using such a file to act as defaults in case a user wants to override
|
||
# only some of the settings themselves, make it easy to modify settings
|
||
# post-build so that they immediately apply to the Makefile instead of having to
|
||
# regenerate it, and allow for easy copying of the settings for anyone else who
|
||
# may want to use them.
|
||
f"--config {os.fsdecode(HERE / 'wasmtime.toml')}"
|
||
)
|
||
default_logdir = pathlib.Path(tempfile.gettempdir())
|
||
|
||
parser = argparse.ArgumentParser()
|
||
subcommands = parser.add_subparsers(dest="subcommand")
|
||
build = subcommands.add_parser("build", help="Build everything")
|
||
configure_build = subcommands.add_parser(
|
||
"configure-build-python", help="Run `configure` for the build Python"
|
||
)
|
||
make_build = subcommands.add_parser(
|
||
"make-build-python", help="Run `make` for the build Python"
|
||
)
|
||
configure_host = subcommands.add_parser(
|
||
"configure-host",
|
||
help="Run `configure` for the "
|
||
"host/WASI (pydebug builds "
|
||
"are inferred from the build "
|
||
"Python)",
|
||
)
|
||
make_host = subcommands.add_parser(
|
||
"make-host", help="Run `make` for the host/WASI"
|
||
)
|
||
subcommands.add_parser(
|
||
"clean", help="Delete files and directories created by this script"
|
||
)
|
||
for subcommand in (
|
||
build,
|
||
configure_build,
|
||
make_build,
|
||
configure_host,
|
||
make_host,
|
||
):
|
||
subcommand.add_argument(
|
||
"--quiet",
|
||
action="store_true",
|
||
default=False,
|
||
dest="quiet",
|
||
help="Redirect output from subprocesses to a log file",
|
||
)
|
||
subcommand.add_argument(
|
||
"--logdir",
|
||
type=pathlib.Path,
|
||
default=default_logdir,
|
||
help=f"Directory to store log files; defaults to {default_logdir}",
|
||
)
|
||
for subcommand in configure_build, configure_host:
|
||
subcommand.add_argument(
|
||
"--clean",
|
||
action="store_true",
|
||
default=False,
|
||
dest="clean",
|
||
help="Delete any relevant directories before building",
|
||
)
|
||
for subcommand in build, configure_build, configure_host:
|
||
subcommand.add_argument(
|
||
"args", nargs="*", help="Extra arguments to pass to `configure`"
|
||
)
|
||
for subcommand in build, configure_host:
|
||
subcommand.add_argument(
|
||
"--wasi-sdk",
|
||
type=pathlib.Path,
|
||
dest="wasi_sdk_path",
|
||
default=default_wasi_sdk,
|
||
help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
|
||
)
|
||
subcommand.add_argument(
|
||
"--host-runner",
|
||
action="store",
|
||
default=default_host_runner,
|
||
dest="host_runner",
|
||
help="Command template for running the WASI host; defaults to "
|
||
f"`{default_host_runner}`",
|
||
)
|
||
for subcommand in build, configure_host, make_host:
|
||
subcommand.add_argument(
|
||
"--host-triple",
|
||
action="store",
|
||
default=default_host_triple,
|
||
help="The target triple for the WASI host build; "
|
||
f"defaults to {default_host_triple}",
|
||
)
|
||
|
||
context = parser.parse_args()
|
||
context.init_dir = pathlib.Path().absolute()
|
||
|
||
dispatch = {
|
||
"configure-build-python": configure_build_python,
|
||
"make-build-python": make_build_python,
|
||
"configure-host": configure_wasi_python,
|
||
"make-host": make_wasi_python,
|
||
"build": build_all,
|
||
"clean": clean_contents,
|
||
}
|
||
dispatch[context.subcommand](context)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|