Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-04-14 07:41:00 +00:00
Code Issues Projects Releases Packages Wiki Activity

gh-145219: Cache Emscripten build dependencies, add install-emscripten (#145664)

Browse source 
Modifies the Emscripten build script to allow for caching of dependencies, and
for automated installation of new EMSDK versions.

Co-authored-by: Russell Keith-Magee <russell@keith-magee.com>
This commit is contained in:
Hood Chatham 2026-03-11 01:43:27 +01:00 • committed by GitHub
parent 5197ecb5e4
commit ebb150e76a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 184 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

214
Tools/wasm/emscripten/__main__.py
View file

@ -4,6 +4,7 @@
import contextlib
import functools
import hashlib
import json
import os
import shutil
import subprocess
@ -14,6 +15,8 @@
from textwrap import dedent
from urllib.request import urlopen
import tomllib
try:
from os import process_cpu_count as cpu_count
except ImportError:
@ -22,25 +25,51 @@
EMSCRIPTEN_DIR = Path(__file__).parent
CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
EMSCRIPTEN_VERSION_FILE = EMSCRIPTEN_DIR / "emscripten_version.txt"
CONFIG_FILE = EMSCRIPTEN_DIR / "config.toml"
DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build"
HOST_TRIPLE = "wasm32-emscripten"
def get_build_paths(cross_build_dir=None):
@functools.cache
def load_config_toml():
with CONFIG_FILE.open("rb") as file:
return tomllib.load(file)
@functools.cache
def required_emscripten_version():
return load_config_toml()["emscripten-version"]
@functools.cache
def emsdk_cache_root(emsdk_cache):
required_version = required_emscripten_version()
return Path(emsdk_cache).absolute() / required_version
@functools.cache
def emsdk_activate_path(emsdk_cache):
return emsdk_cache_root(emsdk_cache) / "emsdk/emsdk_env.sh"
def get_build_paths(cross_build_dir=None, emsdk_cache=None):
"""Compute all build paths from the given cross-build directory."""
if cross_build_dir is None:
cross_build_dir = DEFAULT_CROSS_BUILD_DIR
cross_build_dir = Path(cross_build_dir).absolute()
host_triple_dir = cross_build_dir / HOST_TRIPLE
prefix_dir = host_triple_dir / "prefix"
if emsdk_cache:
prefix_dir = emsdk_cache_root(emsdk_cache) / "prefix"
return {
"cross_build_dir": cross_build_dir,
"native_build_dir": cross_build_dir / "build",
"host_triple_dir": host_triple_dir,
"host_build_dir": host_triple_dir / "build",
"host_dir": host_triple_dir / "build" / "python",
"prefix_dir": host_triple_dir / "prefix",
"prefix_dir": prefix_dir,
}
@ -48,22 +77,10 @@ def get_build_paths(cross_build_dir=None):
LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n"
@functools.cache
def get_required_emscripten_version():
"""Read the required emscripten version from emscripten_version.txt."""
return EMSCRIPTEN_VERSION_FILE.read_text().strip()
@functools.cache
def get_emsdk_activate_path(emsdk_cache):
required_version = get_required_emscripten_version()
return Path(emsdk_cache) / required_version / "emsdk_env.sh"
def validate_emsdk_version(emsdk_cache):
"""Validate that the emsdk cache contains the required emscripten version."""
required_version = get_required_emscripten_version()
emsdk_env = get_emsdk_activate_path(emsdk_cache)
required_version = required_emscripten_version()
emsdk_env = emsdk_activate_path(emsdk_cache)
if not emsdk_env.is_file():
print(
f"Required emscripten version {required_version} not found in {emsdk_cache}",
@ -90,7 +107,7 @@ def get_emsdk_environ(emsdk_cache):
[
"bash",
"-c",
f"EMSDK_QUIET=1 source {get_emsdk_activate_path(emsdk_cache)} && env",
f"EMSDK_QUIET=1 source {emsdk_activate_path(emsdk_cache)} && env",
],
text=True,
)
@ -207,6 +224,35 @@ def build_python_path(context):
return binary
def install_emscripten(context):
emsdk_cache = context.emsdk_cache
if emsdk_cache is None:
print("install-emscripten requires --emsdk-cache", file=sys.stderr)
sys.exit(1)
version = required_emscripten_version()
emsdk_target = emsdk_cache_root(emsdk_cache) / "emsdk"
if emsdk_target.exists():
if not context.quiet:
print(f"Emscripten version {version} already installed")
return
if not context.quiet:
print(f"Installing emscripten version {version}")
emsdk_target.mkdir(parents=True)
call(
[
"git",
"clone",
"https://github.com/emscripten-core/emsdk.git",
emsdk_target,
],
quiet=context.quiet,
)
call([emsdk_target / "emsdk", "install", version], quiet=context.quiet)
call([emsdk_target / "emsdk", "activate", version], quiet=context.quiet)
if not context.quiet:
print(f"Installed emscripten version {version}")
@subdir("native_build_dir", clean_ok=True)
def configure_build_python(context, working_dir):
"""Configure the build/host Python."""
@ -258,35 +304,87 @@ def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
shutil.unpack_archive(tmp_file.name, working_dir)
def should_build_library(prefix, name, config, quiet):
cached_config = prefix / (name + ".json")
if not cached_config.exists():
if not quiet:
print(
f"No cached build of {name} version {config['version']} found, building"
)
return True
try:
with cached_config.open("rb") as f:
cached_config = json.load(f)
except json.JSONDecodeError:
if not quiet:
print(f"Cached data for {name} invalid, rebuilding")
return True
if config == cached_config:
if not quiet:
print(
f"Found cached build of {name} version {config['version']}, not rebuilding"
)
return False
if not quiet:
print(
f"Found cached build of {name} version {config['version']} but it's out of date, rebuilding"
)
return True
def write_library_config(prefix, name, config, quiet):
cached_config = prefix / (name + ".json")
with cached_config.open("w") as f:
json.dump(config, f)
if not quiet:
print(f"Succeded building {name}, wrote config to {cached_config}")
@subdir("host_build_dir", clean_ok=True)
def make_emscripten_libffi(context, working_dir):
ver = "3.4.6"
libffi_dir = working_dir / f"libffi-{ver}"
prefix = context.build_paths["prefix_dir"]
libffi_config = load_config_toml()["libffi"]
if not should_build_library(
prefix, "libffi", libffi_config, context.quiet
):
return
url = libffi_config["url"]
version = libffi_config["version"]
shasum = libffi_config["shasum"]
libffi_dir = working_dir / f"libffi-{version}"
shutil.rmtree(libffi_dir, ignore_errors=True)
download_and_unpack(
working_dir,
f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz",
"b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e",
url.format(version=version),
shasum,
)
call(
[EMSCRIPTEN_DIR / "make_libffi.sh"],
env=updated_env(
{"PREFIX": context.build_paths["prefix_dir"]}, context.emsdk_cache
),
env=updated_env({"PREFIX": prefix}, context.emsdk_cache),
cwd=libffi_dir,
quiet=context.quiet,
)
write_library_config(prefix, "libffi", libffi_config, context.quiet)
@subdir("host_build_dir", clean_ok=True)
def make_mpdec(context, working_dir):
ver = "4.0.1"
mpdec_dir = working_dir / f"mpdecimal-{ver}"
prefix = context.build_paths["prefix_dir"]
mpdec_config = load_config_toml()["mpdec"]
if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet):
return
url = mpdec_config["url"]
version = mpdec_config["version"]
shasum = mpdec_config["shasum"]
mpdec_dir = working_dir / f"mpdecimal-{version}"
shutil.rmtree(mpdec_dir, ignore_errors=True)
download_and_unpack(
working_dir,
f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz",
"96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
url.format(version=version),
shasum,
)
call(
[
@ -294,7 +392,7 @@ def make_mpdec(context, working_dir):
mpdec_dir / "configure",
"CFLAGS=-fPIC",
"--prefix",
context.build_paths["prefix_dir"],
prefix,
"--disable-shared",
],
cwd=mpdec_dir,
@ -306,6 +404,7 @@ def make_mpdec(context, working_dir):
cwd=mpdec_dir,
quiet=context.quiet,
)
write_library_config(prefix, "mpdec", mpdec_config, context.quiet)
@subdir("host_dir", clean_ok=True)
@ -436,16 +535,24 @@ def make_emscripten_python(context, working_dir):
subprocess.check_call([exec_script, "--version"])
def build_all(context):
"""Build everything."""
steps = [
configure_build_python,
make_build_python,
make_emscripten_libffi,
make_mpdec,
configure_emscripten_python,
make_emscripten_python,
]
def build_target(context):
"""Build one or more targets."""
steps = []
if context.target in {"all"}:
steps.append(install_emscripten)
if context.target in {"build", "all"}:
steps.extend([
configure_build_python,
make_build_python,
])
if context.target in {"host", "all"}:
steps.extend([
make_emscripten_libffi,
make_mpdec,
configure_emscripten_python,
make_emscripten_python,
])
for step in steps:
step(context)
@ -475,7 +582,22 @@ def main():
parser = argparse.ArgumentParser()
subcommands = parser.add_subparsers(dest="subcommand")
install_emscripten_cmd = subcommands.add_parser(
"install-emscripten",
help="Install the appropriate version of Emscripten",
)
build = subcommands.add_parser("build", help="Build everything")
build.add_argument(
"target",
nargs="?",
default="all",
choices=["all", "host", "build"],
help=(
"What should be built. 'build' for just the build platform, or "
"'host' for the host platform, or 'all' for both. Defaults to 'all'."
),
)
configure_build = subcommands.add_parser(
"configure-build-python", help="Run `configure` for the build Python"
)
@ -512,6 +634,7 @@ def main():
)
for subcommand in (
install_emscripten_cmd,
build,
configure_build,
make_libffi_cmd,
@ -568,22 +691,25 @@ def main():
context = parser.parse_args()
context.build_paths = get_build_paths(context.cross_build_dir)
if context.emsdk_cache:
if context.emsdk_cache and context.subcommand != "install-emscripten":
validate_emsdk_version(context.emsdk_cache)
context.emsdk_cache = Path(context.emsdk_cache).absolute()
else:
print("Build will use EMSDK from current environment.")
context.build_paths = get_build_paths(
context.cross_build_dir, context.emsdk_cache
)
dispatch = {
"install-emscripten": install_emscripten,
"make-libffi": make_emscripten_libffi,
"make-mpdec": make_mpdec,
"configure-build-python": configure_build_python,
"make-build-python": make_build_python,
"configure-host": configure_emscripten_python,
"make-host": make_emscripten_python,
"build": build_all,
"build": build_target,
"clean": clean_contents,
}

14
Tools/wasm/emscripten/config.toml Normal file
View file

@ -0,0 +1,14 @@
# Any data that can vary between Python versions is to be kept in this file.
# This allows for blanket copying of the Emscripten build code between supported
# Python versions.
emscripten-version = "4.0.12"
[libffi]
url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz"
version = "3.4.6"
shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e"
[mpdec]
url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz"
version = "4.0.1"
shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8"

1
Tools/wasm/emscripten/emscripten_version.txt
View file

@ -1 +0,0 @@
4.0.12