[3.14] gh-146197: Add Emscripten to CI (GH-146198) (GH-146331)

(cherry picked from commit c94048be02)

Co-authored-by: Hood Chatham <roberthoodchatham@gmail.com>
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
Co-authored-by: Victor Stinner <vstinner@python.org>

.github/workflows/build.yml

@@ -386,6 +386,12 @@ jobs:
       - name: Build and test
         run: python3 Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5'
 
+  build-emscripten:
+    name: 'Emscripten'
+    needs: build-context
+    if: needs.build-context.outputs.run-emscripten == 'true'
+    uses: ./.github/workflows/reusable-emscripten.yml
+
   build-wasi:
     name: 'WASI'
     needs: build-context
@@ -664,6 +670,7 @@ jobs:
     - build-ubuntu
     - build-ubuntu-ssltests
     - build-ios
+    - build-emscripten
     - build-wasi
     - test-hypothesis
     - build-asan
@@ -678,6 +685,7 @@ jobs:
       with:
         allowed-failures: >-
           build-android,
+          build-emscripten,
           build-windows-msi,
           build-ubuntu-ssltests,
           test-hypothesis,
@@ -714,5 +722,6 @@ jobs:
           }}
           ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }}
           ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }}
+          ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }}
           ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }}
         jobs: ${{ toJSON(needs) }}

.github/workflows/reusable-context.yml

@@ -41,6 +41,9 @@ on:  # yamllint disable-line rule:truthy
       run-ubuntu:
         description: Whether to run the Ubuntu tests
         value: ${{ jobs.compute-changes.outputs.run-ubuntu }}  # bool
+      run-emscripten:
+        description: Whether to run the Emscripten tests
+        value: ${{ jobs.compute-changes.outputs.run-emscripten }}  # bool
       run-wasi:
         description: Whether to run the WASI tests
         value: ${{ jobs.compute-changes.outputs.run-wasi }}  # bool
@@ -65,6 +68,7 @@ jobs:
       run-macos: ${{ steps.changes.outputs.run-macos }}
       run-tests: ${{ steps.changes.outputs.run-tests }}
       run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }}
+      run-emscripten: ${{ steps.changes.outputs.run-emscripten }}
       run-wasi: ${{ steps.changes.outputs.run-wasi }}
       run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }}
       run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }}

.github/workflows/reusable-emscripten.yml

@@ -0,0 +1,74 @@
+name: Reusable Emscripten
+
+on:
+  workflow_call:
+
+env:
+  FORCE_COLOR: 1
+
+jobs:
+  build-emscripten-reusable:
+    name: 'build and test'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        persist-credentials: false
+    - name: "Read Emscripten config"
+      id: emscripten-config
+      shell: python
+      run: |
+        import hashlib
+        import json
+        import os
+        import tomllib
+        from pathlib import Path
+
+        config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text())
+        h = hashlib.sha256()
+        h.update(json.dumps(config["dependencies"], sort_keys=True).encode())
+        h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes())
+        h.update(b'1') # Update to explicitly bust cache
+        emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache"
+        with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+            f.write(f"emscripten-version={config['emscripten-version']}\n")
+            f.write(f"node-version={config['node-version']}\n")
+            f.write(f"deps-hash={h.hexdigest()}\n")
+        with open(os.environ["GITHUB_ENV"], "a") as f:
+            f.write(f"EMSDK_CACHE={emsdk_cache}\n")
+    - name: "Install Node.js"
+      uses: actions/setup-node@v6
+      with:
+        node-version: ${{ steps.emscripten-config.outputs.node-version }}
+    - name: "Cache Emscripten SDK"
+      id: emsdk-cache
+      uses: actions/cache@v5
+      with:
+        path: ${{ env.EMSDK_CACHE }}
+        key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }}
+        restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}
+    - name: "Install Python"
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.x'
+    - name: "Runner image version"
+      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
+    - name: "Install Emscripten"
+      run: python3 Platforms/emscripten install-emscripten
+    - name: "Configure build Python"
+      run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug
+    - name: "Make build Python"
+      run: python3 Platforms/emscripten make-build-python
+    - name: "Make dependencies"
+      run: >-
+        python3 Platforms/emscripten make-dependencies
+        ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }}
+    - name: "Configure host Python"
+      run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache
+    - name: "Make host Python"
+      run: python3 Platforms/emscripten make-host
+    - name: "Display build info"
+      run: python3 Platforms/emscripten run --pythoninfo
+    - name: "Test"
+      run: python3 Platforms/emscripten run --test

Platforms/emscripten/__main__.py

@@ -350,11 +350,18 @@ def write_library_config(prefix, name, config, quiet):
 def make_emscripten_libffi(context, working_dir):
     validate_emsdk_version(context.emsdk_cache)
     prefix = context.build_paths["prefix_dir"]
-    libffi_config = load_config_toml()["libffi"]
+    libffi_config = load_config_toml()["dependencies"]["libffi"]
+    with open(EMSCRIPTEN_DIR / "make_libffi.sh", "rb") as f:
+        libffi_config["make_libffi_shasum"] = hashlib.file_digest(f, "sha256").hexdigest()
     if not should_build_library(
         prefix, "libffi", libffi_config, context.quiet
     ):
         return
+
+    if context.check_up_to_date:
+        print("libffi out of date, expected to be up to date", file=sys.stderr)
+        sys.exit(1)
+
     url = libffi_config["url"]
     version = libffi_config["version"]
     shasum = libffi_config["shasum"]
@@ -378,10 +385,14 @@ def make_emscripten_libffi(context, working_dir):
 def make_mpdec(context, working_dir):
     validate_emsdk_version(context.emsdk_cache)
     prefix = context.build_paths["prefix_dir"]
-    mpdec_config = load_config_toml()["mpdec"]
+    mpdec_config = load_config_toml()["dependencies"]["mpdec"]
     if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet):
         return
 
+    if context.check_up_to_date:
+        print("libmpdec out of date, expected to be up to date", file=sys.stderr)
+        sys.exit(1)
+
     url = mpdec_config["url"]
     version = mpdec_config["version"]
     shasum = mpdec_config["shasum"]
@@ -680,6 +691,14 @@ def main():
         help="Build all static library dependencies",
     )
 
+    for cmd in [make_mpdec_cmd, make_libffi_cmd, make_dependencies_cmd]:
+        cmd.add_argument(
+            "--check-up-to-date",
+            action="store_true",
+            default=False,
+            help=("If passed, will fail if dependency is out of date"),
+        )
+
     make_build = subcommands.add_parser(
         "make-build-python", help="Run `make` for the build Python"
     )
@@ -707,7 +726,7 @@ def main():
         help=(
             "Add the default test arguments to the beginning of the command. "
             "Default arguments loaded from Platforms/emscripten/config.toml"
-        )
+        ),
     )
     run.add_argument(
         "--pythoninfo",
@@ -721,7 +740,7 @@ def main():
         help=(
             "Arguments to pass to the emscripten Python "
             "(use '--' to separate from run options)",
-        )
+        ),
     )
     add_cross_build_dir_option(run)
 

Platforms/emscripten/config.toml

@@ -15,12 +15,12 @@ pythoninfo-args = [
     "-m", "test.pythoninfo",
 ]
 
-[libffi]
+[dependencies.libffi]
 url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz"
 version = "3.4.6"
 shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e"
 
-[mpdec]
+[dependencies.mpdec]
 url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz"
 version = "4.0.1"
 shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8"

Tools/build/compute-changes.py

@@ -48,6 +48,7 @@
 SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"})
 
 ANDROID_DIRS = frozenset({"Android"})
+EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")})
 IOS_DIRS = frozenset({"Apple", "iOS"})
 MACOS_DIRS = frozenset({"Mac"})
 WASI_DIRS = frozenset({Path("Tools", "wasm")})
@@ -106,6 +107,7 @@ class Outputs:
     run_ci_fuzz: bool = False
     run_ci_fuzz_stdlib: bool = False
     run_docs: bool = False
+    run_emscripten: bool = False
     run_ios: bool = False
     run_macos: bool = False
     run_tests: bool = False
@@ -125,6 +127,7 @@ def compute_changes() -> None:
         # Otherwise, just run the tests
         outputs = Outputs(
             run_android=True,
+            run_emscripten=True,
             run_ios=True,
             run_macos=True,
             run_tests=True,
@@ -194,6 +197,8 @@ def get_file_platform(file: Path) -> str | None:
         return "ios"
     if first_part in ANDROID_DIRS:
         return "android"
+    if len(file.parts) >= 2 and Path(*file.parts[:2]) in EMSCRIPTEN_DIRS:
+        return "emscripten"
     if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: # Tools/wasm/
         return "wasi"
     return None
@@ -242,6 +247,10 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
                 run_tests = True
                 platforms_changed.add("macos")
                 continue
+            if file.name == "reusable-emscripten.yml":
+                run_tests = True
+                platforms_changed.add("emscripten")
+                continue
             if file.name == "reusable-wasi.yml":
                 run_tests = True
                 platforms_changed.add("wasi")
@@ -282,18 +291,21 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
     if run_tests:
         if not has_platform_specific_change or not platforms_changed:
             run_android = True
+            run_emscripten = True
             run_ios = True
             run_macos = True
             run_ubuntu = True
             run_wasi = True
         else:
             run_android = "android" in platforms_changed
+            run_emscripten = "emscripten" in platforms_changed
             run_ios = "ios" in platforms_changed
             run_macos = "macos" in platforms_changed
             run_ubuntu = False
             run_wasi = "wasi" in platforms_changed
     else:
         run_android = False
+        run_emscripten = False
         run_ios = False
         run_macos = False
         run_ubuntu = False
@@ -304,6 +316,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
         run_ci_fuzz=run_ci_fuzz,
         run_ci_fuzz_stdlib=run_ci_fuzz_stdlib,
         run_docs=run_docs,
+        run_emscripten=run_emscripten,
         run_ios=run_ios,
         run_macos=run_macos,
         run_tests=run_tests,