[3.14] GH-139590: Run ruff format on pre-commit for Tools/wasm (GH-139591) (#139744)

Co-authored-by: Savannah Ostrowski <savannahostrowski@gmail.com>
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
This commit is contained in:
Miss Islington (bot) 2025-10-08 17:11:09 +02:00 committed by GitHub
parent a96cde9f80
commit c5ab1b7661
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 323 additions and 157 deletions

View file

@ -34,6 +34,10 @@ repos:
name: Run Ruff (format) on Tools/build/check_warnings.py name: Run Ruff (format) on Tools/build/check_warnings.py
args: [--check, --config=Tools/build/.ruff.toml] args: [--check, --config=Tools/build/.ruff.toml]
files: ^Tools/build/check_warnings.py files: ^Tools/build/check_warnings.py
- id: ruff-format
name: Run Ruff (format) on Tools/wasm/
args: [--check, --config=Tools/wasm/.ruff.toml]
files: ^Tools/wasm/
- repo: https://github.com/psf/black-pre-commit-mirror - repo: https://github.com/psf/black-pre-commit-mirror
rev: 25.9.0 rev: 25.9.0

28
Tools/wasm/.ruff.toml Normal file
View file

@ -0,0 +1,28 @@
extend = "../../.ruff.toml" # Inherit the project-wide settings
[format]
preview = true
docstring-code-format = true
[lint]
select = [
"C4", # flake8-comprehensions
"E", # pycodestyle
"F", # pyflakes
"I", # isort
"ISC", # flake8-implicit-str-concat
"LOG", # flake8-logging
"PGH", # pygrep-hooks
"PT", # flake8-pytest-style
"PYI", # flake8-pyi
"RUF100", # Ban unused `# noqa` comments
"UP", # pyupgrade
"W", # pycodestyle
"YTT", # flake8-2020
]
ignore = [
"E501", # Line too long
"F541", # f-string without any placeholders
"PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple`
"PYI025", # Use `from collections.abc import Set as AbstractSet`
]

View file

@ -33,7 +33,9 @@
PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix"
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode(
"utf-8"
)
def updated_env(updates={}): def updated_env(updates={}):
@ -45,7 +47,9 @@ def updated_env(updates={}):
# https://reproducible-builds.org/docs/source-date-epoch/ # https://reproducible-builds.org/docs/source-date-epoch/
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
try: try:
epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() epoch = subprocess.check_output(
git_epoch_cmd, encoding="utf-8"
).strip()
env_defaults["SOURCE_DATE_EPOCH"] = epoch env_defaults["SOURCE_DATE_EPOCH"] = epoch
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
pass # Might be building from a tarball. pass # Might be building from a tarball.
@ -79,7 +83,11 @@ def wrapper(context):
terminal_width = 80 terminal_width = 80
print("" * terminal_width) print("" * terminal_width)
print("📁", working_dir) print("📁", working_dir)
if clean_ok and getattr(context, "clean", False) and working_dir.exists(): if (
clean_ok
and getattr(context, "clean", False)
and working_dir.exists()
):
print("🚮 Deleting directory (--clean)...") print("🚮 Deleting directory (--clean)...")
shutil.rmtree(working_dir) shutil.rmtree(working_dir)
@ -128,7 +136,9 @@ def build_python_path():
if not binary.is_file(): if not binary.is_file():
binary = binary.with_suffix(".exe") binary = binary.with_suffix(".exe")
if not binary.is_file(): if not binary.is_file():
raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") raise FileNotFoundError(
f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}"
)
return binary return binary
@ -158,7 +168,8 @@ def make_build_python(context, working_dir):
cmd = [ cmd = [
binary, binary,
"-c", "-c",
"import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", "import sys; "
"print(f'{sys.version_info.major}.{sys.version_info.minor}')",
] ]
version = subprocess.check_output(cmd, encoding="utf-8").strip() version = subprocess.check_output(cmd, encoding="utf-8").strip()
@ -173,7 +184,9 @@ def check_shasum(file: str, expected_shasum: str):
def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete_on_close=False) as tmp_file: with tempfile.NamedTemporaryFile(
suffix=".tar.gz", delete_on_close=False
) as tmp_file:
with urlopen(url) as response: with urlopen(url) as response:
shutil.copyfileobj(response, tmp_file) shutil.copyfileobj(response, tmp_file)
tmp_file.close() tmp_file.close()
@ -186,7 +199,11 @@ def make_emscripten_libffi(context, working_dir):
ver = "3.4.6" ver = "3.4.6"
libffi_dir = working_dir / f"libffi-{ver}" libffi_dir = working_dir / f"libffi-{ver}"
shutil.rmtree(libffi_dir, ignore_errors=True) shutil.rmtree(libffi_dir, ignore_errors=True)
download_and_unpack(working_dir, f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e") download_and_unpack(
working_dir,
f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz",
"b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e",
)
call( call(
[EMSCRIPTEN_DIR / "make_libffi.sh"], [EMSCRIPTEN_DIR / "make_libffi.sh"],
env=updated_env({"PREFIX": PREFIX_DIR}), env=updated_env({"PREFIX": PREFIX_DIR}),
@ -200,7 +217,11 @@ def make_mpdec(context, working_dir):
ver = "4.0.1" ver = "4.0.1"
mpdec_dir = working_dir / f"mpdecimal-{ver}" mpdec_dir = working_dir / f"mpdecimal-{ver}"
shutil.rmtree(mpdec_dir, ignore_errors=True) shutil.rmtree(mpdec_dir, ignore_errors=True)
download_and_unpack(working_dir, f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8") download_and_unpack(
working_dir,
f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz",
"96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
)
call( call(
[ [
"emconfigure", "emconfigure",
@ -214,10 +235,7 @@ def make_mpdec(context, working_dir):
quiet=context.quiet, quiet=context.quiet,
) )
call( call(
[ ["make", "install"],
"make",
"install"
],
cwd=mpdec_dir, cwd=mpdec_dir,
quiet=context.quiet, quiet=context.quiet,
) )
@ -226,17 +244,15 @@ def make_mpdec(context, working_dir):
@subdir(HOST_DIR, clean_ok=True) @subdir(HOST_DIR, clean_ok=True)
def configure_emscripten_python(context, working_dir): def configure_emscripten_python(context, working_dir):
"""Configure the emscripten/host build.""" """Configure the emscripten/host build."""
config_site = os.fsdecode( config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten")
EMSCRIPTEN_DIR / "config.site-wasm32-emscripten"
)
emscripten_build_dir = working_dir.relative_to(CHECKOUT) emscripten_build_dir = working_dir.relative_to(CHECKOUT)
python_build_dir = NATIVE_BUILD_DIR / "build" python_build_dir = NATIVE_BUILD_DIR / "build"
lib_dirs = list(python_build_dir.glob("lib.*")) lib_dirs = list(python_build_dir.glob("lib.*"))
assert ( assert len(lib_dirs) == 1, (
len(lib_dirs) == 1 f"Expected a single lib.* directory in {python_build_dir}"
), f"Expected a single lib.* directory in {python_build_dir}" )
lib_dir = os.fsdecode(lib_dirs[0]) lib_dir = os.fsdecode(lib_dirs[0])
pydebug = lib_dir.endswith("-pydebug") pydebug = lib_dir.endswith("-pydebug")
python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
@ -290,7 +306,9 @@ def configure_emscripten_python(context, working_dir):
quiet=context.quiet, quiet=context.quiet,
) )
shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs") shutil.copy(
EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs"
)
node_entry = working_dir / "node_entry.mjs" node_entry = working_dir / "node_entry.mjs"
exec_script = working_dir / "python.sh" exec_script = working_dir / "python.sh"
@ -383,13 +401,15 @@ def main():
subcommands = parser.add_subparsers(dest="subcommand") subcommands = parser.add_subparsers(dest="subcommand")
build = subcommands.add_parser("build", help="Build everything") build = subcommands.add_parser("build", help="Build everything")
configure_build = subcommands.add_parser( configure_build = subcommands.add_parser(
"configure-build-python", help="Run `configure` for the " "build Python" "configure-build-python", help="Run `configure` for the build Python"
) )
make_mpdec_cmd = subcommands.add_parser( make_mpdec_cmd = subcommands.add_parser(
"make-mpdec", help="Clone mpdec repo, configure and build it for emscripten" "make-mpdec",
help="Clone mpdec repo, configure and build it for emscripten",
) )
make_libffi_cmd = subcommands.add_parser( make_libffi_cmd = subcommands.add_parser(
"make-libffi", help="Clone libffi repo, configure and build it for emscripten" "make-libffi",
help="Clone libffi repo, configure and build it for emscripten",
) )
make_build = subcommands.add_parser( make_build = subcommands.add_parser(
"make-build-python", help="Run `make` for the build Python" "make-build-python", help="Run `make` for the build Python"
@ -457,7 +477,11 @@ def main():
if not context.subcommand: if not context.subcommand:
# No command provided, display help and exit # No command provided, display help and exit
print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) print(
"Expected one of",
", ".join(sorted(dispatch.keys())),
file=sys.stderr,
)
parser.print_help(sys.stderr) parser.print_help(sys.stderr)
sys.exit(1) sys.exit(1)
dispatch[context.subcommand](context) dispatch[context.subcommand](context)

View file

@ -19,6 +19,7 @@
}}); }});
""" """
def prepare_wasm(input_file, output_file, function_name): def prepare_wasm(input_file, output_file, function_name):
# Read the compiled WASM as binary and convert to hex # Read the compiled WASM as binary and convert to hex
wasm_bytes = Path(input_file).read_bytes() wasm_bytes = Path(input_file).read_bytes()
@ -31,9 +32,7 @@ def prepare_wasm(input_file, output_file, function_name):
) )
Path(output_file).write_text(js_content) Path(output_file).write_text(js_content)
print( print(f"Successfully compiled {input_file} and generated {output_file}")
f"Successfully compiled {input_file} and generated {output_file}"
)
return 0 return 0

View file

@ -27,7 +27,9 @@
WASM_STDLIB_ZIP = ( WASM_STDLIB_ZIP = (
WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
) )
WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" WASM_STDLIB = (
WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
)
WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"

View file

@ -6,10 +6,16 @@
description="Start a local webserver with a Python terminal." description="Start a local webserver with a Python terminal."
) )
parser.add_argument( parser.add_argument(
"--port", type=int, default=8000, help="port for the http server to listen on" "--port",
type=int,
default=8000,
help="port for the http server to listen on",
) )
parser.add_argument( parser.add_argument(
"--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" "--bind",
type=str,
default="127.0.0.1",
help="Bind address (empty for all)",
) )

View file

@ -3,8 +3,10 @@
import runpy import runpy
import sys import sys
print("⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " print(
"⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; "
"execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n",
file=sys.stderr) file=sys.stderr,
)
runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__") runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__")

View file

@ -4,6 +4,7 @@
import contextlib import contextlib
import functools import functools
import os import os
try: try:
from os import process_cpu_count as cpu_count from os import process_cpu_count as cpu_count
except ImportError: except ImportError:
@ -17,15 +18,19 @@
CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent
assert (CHECKOUT / "configure").is_file(), "Please update the location of the file" assert (CHECKOUT / "configure").is_file(), (
"Please update the location of the file"
)
CROSS_BUILD_DIR = CHECKOUT / "cross-build" CROSS_BUILD_DIR = CHECKOUT / "cross-build"
# Build platform can also be found via `config.guess`. # Build platform can also be found via `config.guess`.
BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" LOCAL_SETUP_MARKER = (
"# Required to statically build extension modules.").encode("utf-8") "# Generated by Tools/wasm/wasi .\n"
"# Required to statically build extension modules."
).encode("utf-8")
WASI_SDK_VERSION = 24 WASI_SDK_VERSION = 24
@ -42,7 +47,9 @@ def updated_env(updates={}):
# https://reproducible-builds.org/docs/source-date-epoch/ # https://reproducible-builds.org/docs/source-date-epoch/
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
try: try:
epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() epoch = subprocess.check_output(
git_epoch_cmd, encoding="utf-8"
).strip()
env_defaults["SOURCE_DATE_EPOCH"] = epoch env_defaults["SOURCE_DATE_EPOCH"] = epoch
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
pass # Might be building from a tarball. pass # Might be building from a tarball.
@ -63,6 +70,7 @@ def updated_env(updates={}):
def subdir(working_dir, *, clean_ok=False): def subdir(working_dir, *, clean_ok=False):
"""Decorator to change to a working directory.""" """Decorator to change to a working directory."""
def decorator(func): def decorator(func):
@functools.wraps(func) @functools.wraps(func)
def wrapper(context): def wrapper(context):
@ -71,16 +79,20 @@ def wrapper(context):
if callable(working_dir): if callable(working_dir):
working_dir = working_dir(context) working_dir = working_dir(context)
try: try:
tput_output = subprocess.check_output(["tput", "cols"], tput_output = subprocess.check_output(
encoding="utf-8") ["tput", "cols"], encoding="utf-8"
)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
terminal_width = 80 terminal_width = 80
else: else:
terminal_width = int(tput_output.strip()) terminal_width = int(tput_output.strip())
print("" * terminal_width) print("" * terminal_width)
print("📁", working_dir) print("📁", working_dir)
if (clean_ok and getattr(context, "clean", False) and if (
working_dir.exists()): clean_ok
and getattr(context, "clean", False)
and working_dir.exists()
):
print("🚮 Deleting directory (--clean)...") print("🚮 Deleting directory (--clean)...")
shutil.rmtree(working_dir) shutil.rmtree(working_dir)
@ -110,11 +122,14 @@ def call(command, *, context=None, quiet=False, logdir=None, **kwargs):
stdout = None stdout = None
stderr = None stderr = None
else: else:
stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", stdout = tempfile.NamedTemporaryFile(
"w",
encoding="utf-8",
delete=False, delete=False,
dir=logdir, dir=logdir,
prefix="cpython-wasi-", prefix="cpython-wasi-",
suffix=".log") suffix=".log",
)
stderr = subprocess.STDOUT stderr = subprocess.STDOUT
print(f"📝 Logging output to {stdout.name} (--quiet)...") print(f"📝 Logging output to {stdout.name} (--quiet)...")
@ -127,8 +142,9 @@ def build_python_path():
if not binary.is_file(): if not binary.is_file():
binary = binary.with_suffix(".exe") binary = binary.with_suffix(".exe")
if not binary.is_file(): if not binary.is_file():
raise FileNotFoundError("Unable to find `python(.exe)` in " raise FileNotFoundError(
f"{BUILD_DIR}") f"Unable to find `python(.exe)` in {BUILD_DIR}"
)
return binary return binary
@ -136,9 +152,11 @@ def build_python_path():
def build_python_is_pydebug(): def build_python_is_pydebug():
"""Find out if the build Python is a pydebug build.""" """Find out if the build Python is a pydebug build."""
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
result = subprocess.run([build_python_path(), "-c", test], result = subprocess.run(
[build_python_path(), "-c", test],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
)
return bool(result.returncode) return bool(result.returncode)
@ -154,7 +172,7 @@ def configure_build_python(context, working_dir):
print(f"📝 Creating {LOCAL_SETUP} ...") print(f"📝 Creating {LOCAL_SETUP} ...")
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
if context.args: if context.args:
configure.extend(context.args) configure.extend(context.args)
@ -164,13 +182,15 @@ def configure_build_python(context, working_dir):
@subdir(BUILD_DIR) @subdir(BUILD_DIR)
def make_build_python(context, working_dir): def make_build_python(context, working_dir):
"""Make/build the build Python.""" """Make/build the build Python."""
call(["make", "--jobs", str(cpu_count()), "all"], call(["make", "--jobs", str(cpu_count()), "all"], context=context)
context=context)
binary = build_python_path() binary = build_python_path()
cmd = [binary, "-c", cmd = [
binary,
"-c",
"import sys; " "import sys; "
"print(f'{sys.version_info.major}.{sys.version_info.minor}')"] "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
]
version = subprocess.check_output(cmd, encoding="utf-8").strip() version = subprocess.check_output(cmd, encoding="utf-8").strip()
print(f"🎉 {binary} {version}") print(f"🎉 {binary} {version}")
@ -188,8 +208,11 @@ def find_wasi_sdk():
# Starting with WASI SDK 23, the tarballs went from containing a directory named # Starting with WASI SDK 23, the tarballs went from containing a directory named
# ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
# ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
potential_sdks = [path for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") potential_sdks = [
if path.is_dir()] path
for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*")
if path.is_dir()
]
if len(potential_sdks) == 1: if len(potential_sdks) == 1:
return potential_sdks[0] return potential_sdks[0]
elif (default_path := opt_path / "wasi-sdk").is_dir(): elif (default_path := opt_path / "wasi-sdk").is_dir():
@ -200,8 +223,13 @@ def wasi_sdk_env(context):
"""Calculate environment variables for building with wasi-sdk.""" """Calculate environment variables for building with wasi-sdk."""
wasi_sdk_path = context.wasi_sdk_path wasi_sdk_path = context.wasi_sdk_path
sysroot = wasi_sdk_path / "share" / "wasi-sysroot" sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", env = {
"AR": "llvm-ar", "RANLIB": "ranlib"} "CC": "clang",
"CPP": "clang-cpp",
"CXX": "clang++",
"AR": "llvm-ar",
"RANLIB": "ranlib",
}
for env_var, binary_name in list(env.items()): for env_var, binary_name in list(env.items()):
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
@ -212,16 +240,20 @@ def wasi_sdk_env(context):
env["PKG_CONFIG_PATH"] = "" env["PKG_CONFIG_PATH"] = ""
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
map(os.fsdecode, map(
[sysroot / "lib" / "pkgconfig", os.fsdecode,
sysroot / "share" / "pkgconfig"])) [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
)
)
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
env["WASI_SYSROOT"] = os.fsdecode(sysroot) env["WASI_SYSROOT"] = os.fsdecode(sysroot)
env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), env["PATH"] = os.pathsep.join([
os.environ["PATH"]]) os.fsdecode(wasi_sdk_path / "bin"),
os.environ["PATH"],
])
return env return env
@ -230,54 +262,70 @@ def wasi_sdk_env(context):
def configure_wasi_python(context, working_dir): def configure_wasi_python(context, working_dir):
"""Configure the WASI/host build.""" """Configure the WASI/host build."""
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
raise ValueError("WASI-SDK not found; " raise ValueError(
"WASI-SDK not found; "
"download from " "download from "
"https://github.com/WebAssembly/wasi-sdk and/or " "https://github.com/WebAssembly/wasi-sdk and/or "
"specify via $WASI_SDK_PATH or --wasi-sdk") "specify via $WASI_SDK_PATH or --wasi-sdk"
)
config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi") config_site = os.fsdecode(
CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi"
)
wasi_build_dir = working_dir.relative_to(CHECKOUT) wasi_build_dir = working_dir.relative_to(CHECKOUT)
python_build_dir = BUILD_DIR / "build" python_build_dir = BUILD_DIR / "build"
lib_dirs = list(python_build_dir.glob("lib.*")) lib_dirs = list(python_build_dir.glob("lib.*"))
assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" assert len(lib_dirs) == 1, (
f"Expected a single lib.* directory in {python_build_dir}"
)
lib_dir = os.fsdecode(lib_dirs[0]) lib_dir = os.fsdecode(lib_dirs[0])
python_version = lib_dir.rpartition("-")[-1] python_version = lib_dir.rpartition("-")[-1]
sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" sysconfig_data_dir = (
f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
)
# Use PYTHONPATH to include sysconfig data which must be anchored to the # Use PYTHONPATH to include sysconfig data which must be anchored to the
# WASI guest's `/` directory. # WASI guest's `/` directory.
args = {"GUEST_DIR": "/", args = {
"GUEST_DIR": "/",
"HOST_DIR": CHECKOUT, "HOST_DIR": CHECKOUT,
"ENV_VAR_NAME": "PYTHONPATH", "ENV_VAR_NAME": "PYTHONPATH",
"ENV_VAR_VALUE": f"/{sysconfig_data_dir}", "ENV_VAR_VALUE": f"/{sysconfig_data_dir}",
"PYTHON_WASM": working_dir / "python.wasm"} "PYTHON_WASM": working_dir / "python.wasm",
}
# Check dynamically for wasmtime in case it was specified manually via # Check dynamically for wasmtime in case it was specified manually via
# `--host-runner`. # `--host-runner`.
if WASMTIME_HOST_RUNNER_VAR in context.host_runner: if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
if wasmtime := shutil.which("wasmtime"): if wasmtime := shutil.which("wasmtime"):
args[WASMTIME_VAR_NAME] = wasmtime args[WASMTIME_VAR_NAME] = wasmtime
else: else:
raise FileNotFoundError("wasmtime not found; download from " raise FileNotFoundError(
"https://github.com/bytecodealliance/wasmtime") "wasmtime not found; download from "
"https://github.com/bytecodealliance/wasmtime"
)
host_runner = context.host_runner.format_map(args) host_runner = context.host_runner.format_map(args)
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
build_python = os.fsdecode(build_python_path()) build_python = os.fsdecode(build_python_path())
# The path to `configure` MUST be relative, else `python.wasm` is unable # The path to `configure` MUST be relative, else `python.wasm` is unable
# to find the stdlib due to Python not recognizing that it's being # to find the stdlib due to Python not recognizing that it's being
# executed from within a checkout. # executed from within a checkout.
configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), configure = [
os.path.relpath(CHECKOUT / "configure", working_dir),
f"--host={context.host_triple}", f"--host={context.host_triple}",
f"--build={BUILD_DIR.name}", f"--build={BUILD_DIR.name}",
f"--with-build-python={build_python}"] f"--with-build-python={build_python}",
]
if build_python_is_pydebug(): if build_python_is_pydebug():
configure.append("--with-pydebug") configure.append("--with-pydebug")
if context.args: if context.args:
configure.extend(context.args) configure.extend(context.args)
call(configure, call(
configure,
env=updated_env(env_additions | wasi_sdk_env(context)), env=updated_env(env_additions | wasi_sdk_env(context)),
context=context) context=context,
)
python_wasm = working_dir / "python.wasm" python_wasm = working_dir / "python.wasm"
exec_script = working_dir / "python.sh" exec_script = working_dir / "python.sh"
@ -291,9 +339,11 @@ def configure_wasi_python(context, working_dir):
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
def make_wasi_python(context, working_dir): def make_wasi_python(context, working_dir):
"""Run `make` for the WASI/host build.""" """Run `make` for the WASI/host build."""
call(["make", "--jobs", str(cpu_count()), "all"], call(
["make", "--jobs", str(cpu_count()), "all"],
env=updated_env(), env=updated_env(),
context=context) context=context,
)
exec_script = working_dir / "python.sh" exec_script = working_dir / "python.sh"
call([exec_script, "--version"], quiet=False) call([exec_script, "--version"], quiet=False)
@ -305,11 +355,16 @@ def make_wasi_python(context, working_dir):
def build_all(context): def build_all(context):
"""Build everything.""" """Build everything."""
steps = [configure_build_python, make_build_python, configure_wasi_python, steps = [
make_wasi_python] configure_build_python,
make_build_python,
configure_wasi_python,
make_wasi_python,
]
for step in steps: for step in steps:
step(context) step(context)
def clean_contents(context): def clean_contents(context):
"""Delete all files created by this script.""" """Delete all files created by this script."""
if CROSS_BUILD_DIR.exists(): if CROSS_BUILD_DIR.exists():
@ -324,74 +379,111 @@ def clean_contents(context):
def main(): def main():
default_host_triple = "wasm32-wasip1" default_host_triple = "wasm32-wasip1"
default_wasi_sdk = find_wasi_sdk() default_wasi_sdk = find_wasi_sdk()
default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " default_host_runner = (
f"{WASMTIME_HOST_RUNNER_VAR} run "
# Make sure the stack size will work for a pydebug # Make sure the stack size will work for a pydebug
# build. # build.
# Use 16 MiB stack. # Use 16 MiB stack.
"--wasm max-wasm-stack=16777216 " "--wasm max-wasm-stack=16777216 "
# Enable thread support; causes use of preview1. # Enable thread support; causes use of preview1.
#"--wasm threads=y --wasi threads=y " # "--wasm threads=y --wasi threads=y "
# Map the checkout to / to load the stdlib from /Lib. # Map the checkout to / to load the stdlib from /Lib.
"--dir {HOST_DIR}::{GUEST_DIR} " "--dir {HOST_DIR}::{GUEST_DIR} "
# Set PYTHONPATH to the sysconfig data. # Set PYTHONPATH to the sysconfig data.
"--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}"
)
default_logdir = pathlib.Path(tempfile.gettempdir()) default_logdir = pathlib.Path(tempfile.gettempdir())
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
subcommands = parser.add_subparsers(dest="subcommand") subcommands = parser.add_subparsers(dest="subcommand")
build = subcommands.add_parser("build", help="Build everything") build = subcommands.add_parser("build", help="Build everything")
configure_build = subcommands.add_parser("configure-build-python", configure_build = subcommands.add_parser(
help="Run `configure` for the " "configure-build-python", help="Run `configure` for the build Python"
"build Python") )
make_build = subcommands.add_parser("make-build-python", make_build = subcommands.add_parser(
help="Run `make` for the build Python") "make-build-python", help="Run `make` for the build Python"
configure_host = subcommands.add_parser("configure-host", )
configure_host = subcommands.add_parser(
"configure-host",
help="Run `configure` for the " help="Run `configure` for the "
"host/WASI (pydebug builds " "host/WASI (pydebug builds "
"are inferred from the build " "are inferred from the build "
"Python)") "Python)",
make_host = subcommands.add_parser("make-host", )
help="Run `make` for the host/WASI") make_host = subcommands.add_parser(
subcommands.add_parser("clean", help="Delete files and directories " "make-host", help="Run `make` for the host/WASI"
"created by this script") )
for subcommand in build, configure_build, make_build, configure_host, make_host: subcommands.add_parser(
subcommand.add_argument("--quiet", action="store_true", default=False, "clean", help="Delete files and directories created by this script"
)
for subcommand in (
build,
configure_build,
make_build,
configure_host,
make_host,
):
subcommand.add_argument(
"--quiet",
action="store_true",
default=False,
dest="quiet", dest="quiet",
help="Redirect output from subprocesses to a log file") help="Redirect output from subprocesses to a log file",
subcommand.add_argument("--logdir", type=pathlib.Path, default=default_logdir, )
help="Directory to store log files; " subcommand.add_argument(
f"defaults to {default_logdir}") "--logdir",
type=pathlib.Path,
default=default_logdir,
help=f"Directory to store log files; defaults to {default_logdir}",
)
for subcommand in configure_build, configure_host: for subcommand in configure_build, configure_host:
subcommand.add_argument("--clean", action="store_true", default=False, subcommand.add_argument(
"--clean",
action="store_true",
default=False,
dest="clean", dest="clean",
help="Delete any relevant directories before building") help="Delete any relevant directories before building",
)
for subcommand in build, configure_build, configure_host: for subcommand in build, configure_build, configure_host:
subcommand.add_argument("args", nargs="*", subcommand.add_argument(
help="Extra arguments to pass to `configure`") "args", nargs="*", help="Extra arguments to pass to `configure`"
)
for subcommand in build, configure_host: for subcommand in build, configure_host:
subcommand.add_argument("--wasi-sdk", type=pathlib.Path, subcommand.add_argument(
"--wasi-sdk",
type=pathlib.Path,
dest="wasi_sdk_path", dest="wasi_sdk_path",
default=default_wasi_sdk, default=default_wasi_sdk,
help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}") help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
subcommand.add_argument("--host-runner", action="store", )
default=default_host_runner, dest="host_runner", subcommand.add_argument(
"--host-runner",
action="store",
default=default_host_runner,
dest="host_runner",
help="Command template for running the WASI host; defaults to " help="Command template for running the WASI host; defaults to "
f"`{default_host_runner}`") f"`{default_host_runner}`",
)
for subcommand in build, configure_host, make_host: for subcommand in build, configure_host, make_host:
subcommand.add_argument("--host-triple", action="store", subcommand.add_argument(
"--host-triple",
action="store",
default=default_host_triple, default=default_host_triple,
help="The target triple for the WASI host build; " help="The target triple for the WASI host build; "
f"defaults to {default_host_triple}") f"defaults to {default_host_triple}",
)
context = parser.parse_args() context = parser.parse_args()
context.init_dir = pathlib.Path().absolute() context.init_dir = pathlib.Path().absolute()
dispatch = {"configure-build-python": configure_build_python, dispatch = {
"configure-build-python": configure_build_python,
"make-build-python": make_build_python, "make-build-python": make_build_python,
"configure-host": configure_wasi_python, "configure-host": configure_wasi_python,
"make-host": make_wasi_python, "make-host": make_wasi_python,
"build": build_all, "build": build_all,
"clean": clean_contents} "clean": clean_contents,
}
dispatch[context.subcommand](context) dispatch[context.subcommand](context)

View file

@ -21,6 +21,7 @@
./Tools/wasm/wasm_builder.py --clean build build ./Tools/wasm/wasm_builder.py --clean build build
""" """
import argparse import argparse
import enum import enum
import dataclasses import dataclasses
@ -67,7 +68,9 @@
# path to Emscripten SDK config file. # path to Emscripten SDK config file.
# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) EM_CONFIG = pathlib.Path(
os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")
)
EMSDK_MIN_VERSION = (3, 1, 19) EMSDK_MIN_VERSION = (3, 1, 19)
EMSDK_BROKEN_VERSION = { EMSDK_BROKEN_VERSION = {
(3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
@ -261,8 +264,7 @@ def _check_emscripten() -> None:
# git / upstream / tot-upstream installation # git / upstream / tot-upstream installation
version = version[:-4] version = version[:-4]
version_tuple = cast( version_tuple = cast(
Tuple[int, int, int], Tuple[int, int, int], tuple(int(v) for v in version.split("."))
tuple(int(v) for v in version.split("."))
) )
if version_tuple < EMSDK_MIN_VERSION: if version_tuple < EMSDK_MIN_VERSION:
raise ConditionError( raise ConditionError(
@ -518,7 +520,7 @@ def make_cmd(self) -> List[str]:
def getenv(self) -> Dict[str, Any]: def getenv(self) -> Dict[str, Any]:
"""Generate environ dict for platform""" """Generate environ dict for platform"""
env = os.environ.copy() env = os.environ.copy()
if hasattr(os, 'process_cpu_count'): if hasattr(os, "process_cpu_count"):
cpu_count = os.process_cpu_count() cpu_count = os.process_cpu_count()
else: else:
cpu_count = os.cpu_count() cpu_count = os.cpu_count()
@ -596,7 +598,9 @@ def run_py(self, *args: str) -> int:
"""Run Python with hostrunner""" """Run Python with hostrunner"""
self._check_execute() self._check_execute()
return self.run_make( return self.run_make(
"--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" "--eval",
f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}",
"run",
) )
def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None: def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None:
@ -666,9 +670,12 @@ def build_emports(self, force: bool = False) -> None:
# Pre-build libbz2, libsqlite3, libz, and some system libs. # Pre-build libbz2, libsqlite3, libz, and some system libs.
ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
# Multi-threaded sqlite3 has different suffix # Multi-threaded sqlite3 has different suffix
embuilder_cmd.extend( embuilder_cmd.extend([
["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"] "build",
) "bzip2",
"sqlite3-mt" if self.pthreads else "sqlite3",
"zlib",
])
self._run_cmd(embuilder_cmd, cwd=SRCDIR) self._run_cmd(embuilder_cmd, cwd=SRCDIR)
@ -817,7 +824,9 @@ def build_emports(self, force: bool = False) -> None:
# Don't list broken and experimental variants in help # Don't list broken and experimental variants in help
platforms_choices = list(p.name for p in _profiles) + ["cleanall"] platforms_choices = list(p.name for p in _profiles) + ["cleanall"]
platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] platforms_help = list(p.name for p in _profiles if p.support_level) + [
"cleanall"
]
parser.add_argument( parser.add_argument(
"platform", "platform",
metavar="PLATFORM", metavar="PLATFORM",