Conditional support for Cython3.x, CI updates (#808)

* Cython 3.x support needed for Python 3.13
* Move CI to a dynamic matrix
* Use GHA-hosted Apple Silicon runners
* Move Windows builds to cibuildwheel
* Implement rudimentary PEP517 config-settings passthru to setuptools with
  custom in-tree setuptools build backend shim (blocked Windows build
  being moved to cibuildwheel).
* Use build_config.toml to smuggle JSON to cibuildwheel, since it
  trashes JSON via CLI when doing containerized builds.

.github/actions/dynamatrix/action.yml

@@ -0,0 +1,22 @@
+name: Create matrix
+description: Create matrix
+inputs:
+  matrix_yaml:
+    description: input yaml matrix as multiline string; any entry with a bool true `omit` key will be filtered from the output matrix
+    required: true
+outputs:
+  matrix_json:
+    description: filtered matrix as JSON
+    value: ${{ steps.matrix_gen.outputs.matrix_json }}
+
+runs:
+  using: "composite"
+
+  steps:
+    - id: matrix_gen
+      run: |
+        # FIXME: input sanity check to prevent shell injection
+        python3 $GITHUB_ACTION_PATH/matrix_yaml_to_json.py --from-stdin << EOF
+        ${{ inputs.matrix_yaml }}
+        EOF
+      shell: bash

.github/actions/dynamatrix/matrix_yaml_to_json.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import pathlib
+import sys
+import typing as t
+import yaml
+
+from collections.abc import MutableMapping, Sequence
+
+skipped_entries = []
+
+def _filter_omit_entries(value):
+    if isinstance(value, MutableMapping):
+        if (omit_value := value.pop('omit', ...)) is not ...:
+            if omit_value is True or str(omit_value).lower().strip() == 'true':
+                print(f'omitting {value} from matrix')
+                skipped_entries.append(value)
+                return ...
+
+        return {k: v for k, v in ((k, _filter_omit_entries(v)) for k, v in value.items()) if v is not ...}
+
+    if isinstance(value, str):
+        return value
+
+    if isinstance(value, Sequence):
+        return [v for v in (_filter_omit_entries(v) for v in value) if v is not ...]
+
+    return value
+
+def main():
+    p = argparse.ArgumentParser(description='GHA YAML matrix filter')
+    required_grp = p.add_mutually_exclusive_group(required=True)
+    required_grp.add_argument('--from-stdin', action='store_true', help='read input YAML from stdin')
+    required_grp.add_argument('--from-file', type=pathlib.Path, help='read input YAML from file path')
+
+    args = p.parse_args()
+
+    path: pathlib.Path | None
+
+    matrix_yaml: str
+
+    if path := args.from_file:
+        matrix_yaml = path.read_text()
+    elif args.from_stdin:
+        matrix_yaml = sys.stdin.read()
+    else:
+        raise Exception('no source provided for matrix yaml')
+
+    raw_matrix = yaml.safe_load(matrix_yaml)
+    filtered_matrix = _filter_omit_entries(raw_matrix)
+
+    output_matrix_json = json.dumps(filtered_matrix)
+    output_skipped_matrix_json = json.dumps(skipped_entries)
+
+    print(f'filtered matrix: {output_matrix_json}')
+    print(f'skipped entries: {output_skipped_matrix_json}')
+
+    if (gh_output := os.environ.get('GITHUB_OUTPUT')):
+        print('setting step output var matrix_json; skipped_matrix_json...')
+        with pathlib.Path(gh_output).open('a') as env_fd:
+            env_fd.write(f'matrix_json<<__MATRIX_EOF\n{output_matrix_json}\n__MATRIX_EOF\n')
+            env_fd.write(f'skipped_matrix_json<<__MATRIX_EOF\n{output_skipped_matrix_json}\n__MATRIX_EOF\n')
+    else:
+        print("GITHUB_OUTPUT not set; skipping variable output")
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/ci.yaml

@@ -1,8 +1,4 @@
 ---
-# This is the CI workflow (not the artifact build/release workflow). The workflows
-# are split because GHA doesn't support a dynamic/conditional matrix. This workflow
-# has slow jobs and jobs that require private GHA runners (eg, M1 Mac) commented out.
-# Ensure changes are synced with manual_artifact_build.yaml.
 name: PyYAML CI
 
 on:
@@ -10,25 +6,52 @@ on:
   pull_request:
     types: [opened, synchronize, reopened]
   workflow_dispatch:
+    inputs:
+      libyaml_repo:
+        description: Repo URL for libyaml
+        type: string
+        default: https://github.com/yaml/libyaml
+      libyaml_ref:
+        description: Tag/branch ref for libyaml
+        type: string
+        default: 0.2.5
+      skip_artifact_upload:
+        description: Skip most artifact uploads?
+        type: boolean
+        default: true
+      skip_ci_redundant_jobs:
+        description: Skip redundant jobs for CI?
+        type: boolean
+        default: true
+      skip_slow_jobs:
+        description: Skip slow (emulated) jobs for CI?
+        type: boolean
+        default: true
 
 env:
-  LIBYAML_REPO: https://github.com/yaml/libyaml
+  LIBYAML_REPO: ${{ inputs.libyaml_repo || 'https://github.com/yaml/libyaml' }}  # FIXME: can we ref the input.default value?
-  LIBYAML_REF: 0.2.5
+  LIBYAML_REF: ${{ inputs.libyaml_ref || '0.2.5' }}  # FIXME: can we ref the input.default value?
+  skip_ci_redundant_jobs: ${{ inputs.skip_ci_redundant_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }}
+  skip_slow_jobs: ${{ inputs.skip_slow_jobs || github.event_name == 'pull_request' || github.event_name == 'push' }}
+  skip_artifact_upload: ${{ (github.event_name == 'workflow_dispatch' && inputs.skip_artifact_upload) || github.event_name != 'workflow_dispatch' }}
 
 jobs:
   python_sdist:
     name: pyyaml sdist
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    outputs:
+      artifact_name: ${{ steps.build_sdist.outputs.artifact_name }}
     steps:
     - name: Checkout PyYAML
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Install a python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.x
 
     - name: Build sdist
+      id: build_sdist
       env:
         PYYAML_FORCE_CYTHON: 1
         PYYAML_FORCE_LIBYAML: 0
@@ -36,64 +59,60 @@ jobs:
         python -V
         python -m pip install build
 
-        python -m build .
+        python -m build -s .
 
-        # Ensure exactly one artifact was produced.
+        echo "artifact_name=$(ls ./dist)" >> "$GITHUB_OUTPUT"
-        [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || {
-          echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'."
-          exit 1
-        }
-
-    - name: Test sdist
-      run: |
-        # Install some libyaml headers.
-        # TODO Should we smoke test the sdist against the libyaml we built?
-        sudo apt update
-        sudo apt install libyaml-dev -y
-
-        # Ensure Cython is not present so we use only what's in the sdist.
-        python -m pip uninstall Cython -y || true
-
-        # Pass no extra args.
-        # We should auto-install with libyaml since it's present.
-        python -m pip install dist/*.tar.gz -v
-
-        python packaging/build/smoketest.py
 
     - name: Upload sdist artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: dist
+        name: ${{ steps.build_sdist.outputs.artifact_name }}
-        path: dist/*.tar.gz
+        path: dist/${{ steps.build_sdist.outputs.artifact_name }}
+        if-no-files-found: error
+      # always upload the sdist artifact- all the wheel build jobs require it
 
+  make_linux_libyaml_matrix:
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: make a matrix
+      id: make_matrix
+      uses: ./.github/actions/dynamatrix
+      with:
+        matrix_yaml: |
+          include:
+          - { platform: manylinux1, arch: x86_64 }
+          - { platform: manylinux2014, arch: x86_64 }
+          - { platform: manylinux2014, arch: aarch64, omit: ${{ env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, omit: ${{ env.skip_slow_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64 }
+          - { platform: musllinux_1_1, arch: aarch64, omit: ${{ env.skip_slow_jobs }} }
 
   linux_libyaml:
-    name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}}
+    needs: [make_linux_libyaml_matrix]
-    runs-on: ubuntu-latest
+    name: libyaml ${{ matrix.platform }} ${{ matrix.arch }}
+    runs-on: ubuntu-22.04
     strategy:
-      matrix:
+      fail-fast: false
-        cfg:
+      matrix: ${{ fromJSON(needs.make_linux_libyaml_matrix.outputs.matrix_json) }}
-        - { platform: manylinux1, arch: x86_64 }
-        - { platform: manylinux2014, arch: x86_64 }
-#        - { platform: manylinux2014, arch: aarch64 }
-#        - { platform: manylinux2014, arch: s390x }
-        - { platform: musllinux_1_1, arch: x86_64 }
     env:
-      DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}}
+      DOCKER_IMAGE: quay.io/pypa/${{ matrix.platform }}_${{ matrix.arch }}
     steps:
     - name: Check cached libyaml state
       id: cached_libyaml
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: libyaml
-        key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}}
+        key: libyaml_${{ matrix.platform }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
 
     - name: configure docker foreign arch support
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
-      if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true'
+      if: matrix.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true'
 
     - name: Checkout pyyaml
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: steps.cached_libyaml.outputs.cache-hit != 'true'
 
     - name: Build libyaml
@@ -112,52 +131,78 @@ jobs:
         sudo chmod -R a+r ./libyaml/
       if: steps.cached_libyaml.outputs.cache-hit != 'true'
 
+  make_linux_pyyaml_matrix:
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: make a matrix
+      id: make_matrix
+      uses: ./.github/actions/dynamatrix
+      with:
+        matrix_yaml: |
+          include:
+          - { platform: manylinux1, arch: x86_64, spec: cp38 }
+          - { platform: manylinux1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: manylinux2014, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: manylinux2014, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: manylinux2014, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: manylinux2014, arch: x86_64, spec: cp313 }
+          - { platform: manylinux2014, arch: aarch64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: aarch64, spec: cp313, omit: ${{ env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: manylinux2014, arch: s390x, spec: cp313, omit: ${{ env.skip_slow_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64, spec: cp38, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs }} }
+          - { platform: musllinux_1_1, arch: x86_64, spec: cp313 }
+          - { platform: musllinux_1_1, arch: aarch64, spec: cp39, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: musllinux_1_1, arch: aarch64, spec: cp310, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: musllinux_1_1, arch: aarch64, spec: cp311, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: musllinux_1_1, arch: aarch64, spec: cp312, omit: ${{ env.skip_ci_redundant_jobs || env.skip_slow_jobs }} }
+          - { platform: musllinux_1_1, arch: aarch64, spec: cp313, omit: ${{ env.skip_slow_jobs }} }
+
 
   linux_pyyaml:
-    needs: linux_libyaml
+    needs: [python_sdist, linux_libyaml, make_linux_pyyaml_matrix]
-    name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}}
+    name: pyyaml ${{matrix.spec}}-${{matrix.platform}}_${{matrix.arch}}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
-      matrix:
+      fail-fast: false
-        include:
+      matrix: ${{ fromJSON(needs.make_linux_pyyaml_matrix.outputs.matrix_json) }}
-        - { platform: manylinux1, arch: x86_64, spec: cp38 }
-#        - { platform: manylinux1, arch: x86_64, spec: cp39 }
-#        - { platform: manylinux2014, arch: x86_64, spec: cp310 }
-#        - { platform: manylinux2014, arch: x86_64, spec: cp311 }
-        - { platform: manylinux2014, arch: x86_64, spec: cp312 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp38 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp39 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp310 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp311 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp312 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp38 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp39 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp310 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp311 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp312 }
-#        - { platform: musllinux_1_1, arch: x86_64, spec: cp38 }
-#        - { platform: musllinux_1_1, arch: x86_64, spec: cp39 }
-#        - { platform: musllinux_1_1, arch: x86_64, spec: cp310 }
-#        - { platform: musllinux_1_1, arch: x86_64, spec: cp311 }
-        - { platform: musllinux_1_1, arch: x86_64, spec: cp312 }
 
     steps:
-    - name: Checkout PyYAML
+    - name: fetch sdist artifact
-      uses: actions/checkout@v3
+      id: fetch_sdist
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ needs.build_sdist.outputs.artifact_name }}
 
     - name: Fetch cached libyaml
       id: cached_libyaml
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: libyaml
         key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}
         fail-on-cache-miss: true
 
     - name: configure docker foreign arch support
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       if: matrix.arch != 'x86_64'
 
     - name: Build/Test/Package
+      id: build
       env:
         CIBW_ARCHS: all
         # HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first?
@@ -165,25 +210,47 @@ jobs:
         CIBW_BUILD_VERBOSITY: 1
         # containerized Linux builds require explicit CIBW_ENVIRONMENT
         CIBW_ENVIRONMENT: >
-          C_INCLUDE_PATH=libyaml/include
+          LD_LIBRARY_PATH=../libyaml/src/.libs
-          LIBRARY_PATH=libyaml/src/.libs
-          LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
+        CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MANYLINUX_S390X_IMAGE: ${{ matrix.manylinux_img || '' }}
+        CIBW_MUSLLINUX_X86_64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
+        CIBW_MUSLLINUX_I686_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
+        CIBW_MUSLLINUX_AARCH64_IMAGE: ${{ matrix.musllinux_img || 'musllinux_1_1' }}
+        CIBW_PRERELEASE_PYTHONS: 1
         CIBW_TEST_COMMAND: cd {project}; pytest
         CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
+
         python3 -V
-        python3 -m pip install -U --user cibuildwheel
+        python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
-        python3 -m cibuildwheel --platform auto --output-dir dist .
+
+        mkdir pyyaml
+
+        tar zxf ${{ steps.fetch_sdist.outputs.download-path }}/pyyaml*.tar.gz/pyyaml*.tar.gz --strip-components=1 -C pyyaml
+
+        cat << 'EOF' > build_config.toml
+        [tool.cibuildwheel.config-settings]
+        pyyaml_build_config='{"force":1, "library_dirs": ["../libyaml/src/.libs"], "include_dirs": ["../libyaml/include"]}'
+        EOF
+
+        CIBW_BEFORE_BUILD="ls -l {project}" python3 -m cibuildwheel --config-file $(pwd)/build_config.toml --platform auto --output-dir ./dist ./pyyaml
+
+        echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"
+        # FIXME: ensure exactly one artifact
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: dist
+        name: ${{ steps.build.outputs.artifact_name }}
         path: dist/*.whl
         if-no-files-found: error
+      if: ${{ env.skip_artifact_upload != 'true' }}
+
 
   macos_libyaml:
     name: libyaml macos ${{matrix.arch}}
@@ -191,25 +258,25 @@ jobs:
       matrix:
         include:
         - arch: x86_64
-#        - arch: arm64
+          runs-on: macos-13
-#          runs_on: [self-hosted, macOS, arm64]
+          run_wrapper: arch -x86_64 bash --noprofile --norc -eo pipefail {0}
-#          deployment_target: '11.0'
+        - arch: arm64
-#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
+          deployment_target: '11.0'
-#          sdkroot: macosx11.3
+          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
     defaults:
       run:
-        shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
+        shell: ${{ matrix.run_wrapper || 'arch -x86_64 bash --noprofile --norc -eo pipefail {0}' }}
-    runs-on: ${{ matrix.runs_on || 'macos-11' }}
+    runs-on: ${{ matrix.runs-on || 'macos-14' }}
     steps:
     - name: Check cached libyaml state
       id: cached_libyaml
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: libyaml
         key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}}
 
     - name: Checkout PyYAML
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: steps.cached_libyaml.outputs.cache-hit != 'true'
 
     - name: Build libyaml
@@ -218,111 +285,154 @@ jobs:
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         set -eux
-        brew install automake coreutils m4
+        brew install automake coreutils m4 libtool
         bash ./packaging/build/libyaml.sh
         echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)"
       if: steps.cached_libyaml.outputs.cache-hit != 'true'
 
+  make_macos_pyyaml_matrix:
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: make a matrix
+      id: make_matrix
+      uses: ./.github/actions/dynamatrix
+      with:
+        matrix_yaml: |
+          include:
+          - spec: cp38-macosx_x86_64
+            cibw_version: cibuildwheel==2.11.1
+            runs_on: [macos-13]
+            omit: ${{ env.skip_ci_redundant_jobs }}
+          - spec: cp39-macosx_x86_64
+            runs_on: [macos-13]
+            omit: ${{ env.skip_ci_redundant_jobs }}
+          - spec: cp310-macosx_x86_64
+            runs_on: [macos-13]
+            omit: ${{ env.skip_ci_redundant_jobs }}
+          - spec: cp311-macosx_x86_64
+            runs_on: [macos-13]
+            omit: ${{ env.skip_ci_redundant_jobs }}
+          - spec: cp312-macosx_x86_64
+            runs_on: [macos-13]
+            omit: ${{ env.skip_ci_redundant_jobs }}
+          - spec: cp313-macosx_x86_64
+            runs_on: [macos-13]
+            
+          - spec: cp39-macosx_arm64
+            deployment_target: '11.0'
+            arch: arm64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+  
+          - spec: cp310-macosx_arm64
+            deployment_target: '11.0'
+            arch: arm64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+  
+          - spec: cp311-macosx_arm64
+            deployment_target: '11.0'
+            arch: arm64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+  
+          - spec: cp312-macosx_arm64
+            deployment_target: '11.0'
+            arch: arm64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+  
+          - spec: cp313-macosx_arm64
+            deployment_target: '11.0'
+            arch: arm64
+  
 
   macos_pyyaml:
-    needs: macos_libyaml
+    needs: [python_sdist, macos_libyaml, make_macos_pyyaml_matrix]
     name: pyyaml ${{ matrix.spec }}
-    runs-on: ${{ matrix.runs_on || 'macos-11' }}
+    runs-on: ${{ matrix.runs_on || 'macos-14' }}
     defaults:
       run:
         shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
     strategy:
-      matrix:
+      fail-fast: false
-        include:
+      matrix: ${{ fromJSON(needs.make_macos_pyyaml_matrix.outputs.matrix_json) }}
-#        - spec: cp38-macosx_x86_64
-#          cibw_version: cibuildwheel==2.11.1
-#        - spec: cp39-macosx_x86_64
-#        - spec: cp310-macosx_x86_64
-#        - spec: cp311-macosx_x86_64
-        - spec: cp312-macosx_x86_64
-
-#        # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
-#        - spec: cp39-macosx_arm64
-#          deployment_target: '11.0'
-#          runs_on: [self-hosted, macOS, arm64]
-#          arch: arm64
-#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-#          sdkroot: macosx11.3
-#
-#        - spec: cp310-macosx_arm64
-#          deployment_target: '11.0'
-#          runs_on: [self-hosted, macOS, arm64]
-#          arch: arm64
-#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-#          sdkroot: macosx11.3
-#
-#        - spec: cp311-macosx_arm64
-#          deployment_target: '11.0'
-#          runs_on: [self-hosted, macOS, arm64]
-#          arch: arm64
-#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-#          sdkroot: macosx11.3
-#
-#        - spec: cp312-macosx_arm64
-#          deployment_target: '11.0'
-#          runs_on: [self-hosted, macOS, arm64]
-#          arch: arm64
-#          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-#          sdkroot: macosx11.3
-
     steps:
-    - name: Checkout PyYAML
+    - name: fetch sdist artifact
-      uses: actions/checkout@v3
+      id: fetch_sdist
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ needs.build_sdist.outputs.artifact_name }}
 
     - name: Get cached libyaml state
       id: cached_libyaml
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: libyaml
         key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}}
         fail-on-cache-miss: true
 
+    - name: Install python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'  # as of 2024-05, this has to be < 3.12 since the macos-13 runner image's
+                                # built-in virtualenv/pip are pinned to busted versions that fail on newer Pythons
+
     - name: Build/Test/Package
+      id: build
       env:
-        C_INCLUDE_PATH: libyaml/include
+        C_INCLUDE_PATH: ../libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_PRERELEASE_PYTHONS: 1
+        CIBW_TEST_COMMAND: pytest {package}
         CIBW_TEST_REQUIRES: pytest
-        LIBRARY_PATH: libyaml/src/.libs
+        LIBRARY_PATH: ../libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
         PYYAML_FORCE_CYTHON: 1
         PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
+        set -eux
         python3 -V
         python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
-        python3 -m cibuildwheel --platform auto --output-dir dist .
+        mkdir pyyaml
+        
+        tar zxf pyyaml*.tar.gz/pyyaml*.tar.gz --strip-components=1 -C pyyaml
+        
+        cat << 'EOF' > build_config.toml
+        [tool.cibuildwheel.config-settings]
+        pyyaml_build_config='{"force":1, "library_dirs": ["../libyaml/src/.libs"], "include_dirs": ["../libyaml/include"]}'
+        EOF
+        
+        python3 -m cibuildwheel --config-file $(pwd)/build_config.toml --platform auto --output-dir ./dist ./pyyaml
+
+        echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"
+        # FIXME: ensure exactly one artifact
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: dist
+        name: ${{ steps.build.outputs.artifact_name }}
         path: dist/*.whl
         if-no-files-found: error
+      if: ${{ env.skip_artifact_upload != 'true' }}
+
 
   windows_libyaml:
-    name: libyaml ${{matrix.platform}} ${{matrix.arch}}
+    name: libyaml windows ${{ matrix.arch }}
-    runs-on: ${{matrix.platform}}
+    runs-on: ${{ matrix.platform || 'windows-2022' }}
     strategy:
       matrix:
         include:
-        - platform: windows-2019
+        - arch: x64
-          arch: x64
+        - arch: win32
-        - platform: windows-2019
-          arch: win32
     steps:
     - name: Get cached libyaml state
       id: cached_libyaml
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: libyaml
-        key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}
+        key: libyaml_${{ 'windows' }}_${{ matrix.arch }}_${{ env.LIBYAML_REF }}
 
     - name: Build libyaml
       shell: bash
@@ -342,58 +452,63 @@ jobs:
         mkdir libyaml/build
 
         pushd libyaml/build
-        cmake.exe -G "Visual Studio 16 2019" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml ..
+        cmake.exe -G "Visual Studio 17 2022" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml ..
         cmake.exe --build . --config Release
         popd
 
+  make_windows_pyyaml_matrix:
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix_json: ${{ steps.make_matrix.outputs.matrix_json }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: make a matrix
+      id: make_matrix
+      uses: ./.github/actions/dynamatrix
+      with:
+        matrix_yaml: |
+          include:
+          - spec: cp38-win_amd64
+
+          - spec: cp39-win_amd64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp310-win_amd64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp311-win_amd64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp312-win_amd64
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp313-win_amd64
+
+          - spec: cp38-win32
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp39-win32
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp310-win32
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp311-win32
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp312-win32
+            omit: ${{ env.skip_ci_redundant_jobs }}
+
+          - spec: cp313-win32
+            omit: ${{ env.skip_ci_redundant_jobs }}
 
   windows_pyyaml:
-    needs: windows_libyaml
+    needs: [python_sdist, windows_libyaml, make_windows_pyyaml_matrix]
-    name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}}
+    name: pyyaml ${{matrix.spec}}
-    runs-on: ${{matrix.platform}}
+    runs-on: ${{ matrix.runs-on || 'windows-2022' }}
     strategy:
-      matrix:
+      fail-fast: false
-        include:
+      matrix: ${{ fromJSON(needs.make_windows_pyyaml_matrix.outputs.matrix_json) }}
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: 3.8
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: 3.9
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: '3.10'
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: '3.11'
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: '3.12'
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: 3.8
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: 3.9
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: '3.10'
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: '3.11'
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: '3.12'
     steps:
     # autocrlf screws up tests under Windows
     - name: Set git to use LF
@@ -401,48 +516,87 @@ jobs:
         git config --global core.autocrlf false
         git config --global core.eol lf
 
-    - name: Checkout pyyaml
+    - name: fetch sdist artifact
-      uses: actions/checkout@v3
+      id: fetch_sdist
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ needs.build_sdist.outputs.artifact_name }}
 
     - name: Get cached libyaml state
       id: cached_libyaml
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: libyaml
-        key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}}
+        key: libyaml_${{'windows'}}_${{ contains(matrix.spec, 'win_amd64') && 'x64' || 'win32' }}_${{env.LIBYAML_REF}}
         fail-on-cache-miss: true
 
-    - name: Install python ${{ matrix.spec }}
+    - name: Install python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        architecture: ${{ matrix.python_arch }}
+        python-version: 3.x
-        python-version: ${{ matrix.spec }}
 
     - name: Build/Test/Package
-      env:
+      id: build
-        PYYAML_FORCE_CYTHON: 1
-        PYYAML_FORCE_LIBYAML: 1
       shell: bash
+      env:
+        CIBW_BUILD: ${{matrix.spec}}
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_BEFORE_TEST: ls -l {package}
+        CIBW_TEST_COMMAND: pytest {package}
+        CIBW_TEST_REQUIRES: pytest
+        CIBW_PRERELEASE_PYTHONS: 1
+        #CIBW_CONFIG_SETTINGS: |
+        #  pyyaml_build_config='{"include_dirs": ["libyaml/include"], "library_dirs": ["libyaml/build/Release"], "define": [["YAML_DECLARE_STATIC", 1]], "force": 1}'
       run: |
         set -eux
         python -V
-        python -m pip install "Cython<3.0" setuptools wheel
+        python -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
+        mkdir pyyaml
 
-        python setup.py \
+        tar zxf pyyaml*.tar.gz/pyyaml*.tar.gz --strip-components=1 -C pyyaml
-        --with-libyaml build_ext \
-        -I libyaml/include \
-        -L libyaml/build/Release \
-        -D YAML_DECLARE_STATIC \
-        build bdist_wheel
 
-        # run tests on built wheel
+        cat << 'EOF' > build_config.toml
-        python -m pip install dist/*.whl pytest
+        [tool.cibuildwheel.config-settings]
-        python -I -m pytest
+        pyyaml_build_config='{"force":1, "include_dirs": ["../libyaml/include"], "library_dirs": ["../libyaml/build/Release"], "define": [["YAML_DECLARE_STATIC", 1]], "force": 1}'
+        EOF
+
+        python3 -m cibuildwheel --config-file $(pwd)/build_config.toml --platform auto --output-dir ./dist ./pyyaml
+
+        echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT"
+        # FIXME: ensure exactly one artifact
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: dist
+        name: ${{ steps.build.outputs.artifact_name }}
         path: dist/*.whl
         if-no-files-found: error
+      if: ${{ env.skip_artifact_upload != 'true' }}
+
+  merge_artifacts:
+    needs: [python_sdist, macos_pyyaml, linux_pyyaml, windows_pyyaml]
+    runs-on: ubuntu-22.04
+    steps:
+    - name: merge all artifacts
+      uses: actions/upload-artifact/merge@v4
+      with:
+        name: dist
+        delete-merged: true
+      if: ${{ env.skip_artifact_upload != 'true' }}
+
+
+  check:
+    if: always()
+    needs:
+    - python_sdist
+    - linux_pyyaml
+    - macos_pyyaml
+    - windows_pyyaml
+    - merge_artifacts
+    runs-on: ubuntu-latest
+    steps:
+    - name: Verify all previous jobs succeeded (provides a single check to sample for gating purposes)
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}
 ...

.github/workflows/manual_artifact_build.yaml

@@ -1,446 +0,0 @@
----
-# This is the actual artifact build/release workflow. This workflow exists temporarily
-# because GHA doesn't support a dynamic/conditional matrix. Ensure changes are synced with ci.yaml.
-name: PyYAML CI
-
-on:
-#  push:
-#  pull_request:
-#    types: [opened, synchronize, reopened]
-  workflow_dispatch:
-
-env:
-  LIBYAML_REPO: https://github.com/yaml/libyaml
-  LIBYAML_REF: 0.2.5
-
-jobs:
-  python_sdist:
-    name: pyyaml sdist
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout PyYAML
-      uses: actions/checkout@v3
-
-    - name: Install a python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-
-    - name: Build sdist
-      env:
-        PYYAML_FORCE_CYTHON: 1
-        PYYAML_FORCE_LIBYAML: 0
-      run: |
-        python -V
-        python -m pip install build
-
-        python -m build .
-
-        # Ensure exactly one artifact was produced.
-        [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || {
-          echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'."
-          exit 1
-        }
-
-    - name: Test sdist
-      run: |
-        # Install some libyaml headers.
-        # TODO Should we smoke test the sdist against the libyaml we built?
-        sudo apt update
-        sudo apt install libyaml-dev -y
-
-        # Ensure Cython is not present so we use only what's in the sdist.
-        python -m pip uninstall Cython -y || true
-
-        # Pass no extra args.
-        # We should auto-install with libyaml since it's present.
-        python -m pip install dist/*.tar.gz -v
-
-        python packaging/build/smoketest.py
-
-    - name: Upload sdist artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: dist
-        path: dist/*.tar.gz
-
-
-  linux_libyaml:
-    name: libyaml ${{matrix.cfg.arch}} ${{matrix.cfg.platform}}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        cfg:
-        - { platform: manylinux1, arch: x86_64 }
-        - { platform: manylinux2014, arch: x86_64 }
-        - { platform: manylinux2014, arch: aarch64 }
-        - { platform: manylinux2014, arch: s390x }
-        - { platform: musllinux_1_1, arch: x86_64 }
-    env:
-      DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}}
-    steps:
-    - name: Check cached libyaml state
-      id: cached_libyaml
-      uses: actions/cache@v3
-      with:
-        path: libyaml
-        key: libyaml_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}}
-
-    - name: configure docker foreign arch support
-      uses: docker/setup-qemu-action@v2
-      if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true'
-
-    - name: Checkout pyyaml
-      uses: actions/checkout@v3
-      if: steps.cached_libyaml.outputs.cache-hit != 'true'
-
-    - name: Build libyaml
-      run: >
-        docker run --rm
-        --volume "$(pwd):/io"
-        --env LIBYAML_REF
-        --env LIBYAML_REPO
-        --workdir /io
-        "$DOCKER_IMAGE"
-        /io/packaging/build/libyaml.sh
-      if: steps.cached_libyaml.outputs.cache-hit != 'true'
-
-    - name: ensure output is world readable (or cache fill fails with Permission Denied)
-      run: >
-        sudo chmod -R a+r ./libyaml/
-      if: steps.cached_libyaml.outputs.cache-hit != 'true'
-
-
-  linux_pyyaml:
-    needs: linux_libyaml
-    name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.spec}}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-        - { platform: manylinux1, arch: x86_64, spec: cp38 }
-        - { platform: manylinux1, arch: x86_64, spec: cp39 }
-        - { platform: manylinux2014, arch: x86_64, spec: cp310 }
-        - { platform: manylinux2014, arch: x86_64, spec: cp311 }
-        - { platform: manylinux2014, arch: x86_64, spec: cp312 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp38 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp39 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp310 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp311 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp312 }
-        - { platform: manylinux2014, arch: s390x, spec: cp38 }
-        - { platform: manylinux2014, arch: s390x, spec: cp39 }
-        - { platform: manylinux2014, arch: s390x, spec: cp310 }
-        - { platform: manylinux2014, arch: s390x, spec: cp311 }
-        - { platform: manylinux2014, arch: s390x, spec: cp312 }
-        - { platform: musllinux_1_1, arch: x86_64, spec: cp38 }
-        - { platform: musllinux_1_1, arch: x86_64, spec: cp39 }
-        - { platform: musllinux_1_1, arch: x86_64, spec: cp310 }
-        - { platform: musllinux_1_1, arch: x86_64, spec: cp311 }
-        - { platform: musllinux_1_1, arch: x86_64, spec: cp312 }
-
-    steps:
-    - name: Checkout PyYAML
-      uses: actions/checkout@v3
-
-    - name: Fetch cached libyaml
-      id: cached_libyaml
-      uses: actions/cache/restore@v3
-      with:
-        path: libyaml
-        key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}
-        fail-on-cache-miss: true
-
-    - name: configure docker foreign arch support
-      uses: docker/setup-qemu-action@v2
-      if: matrix.arch != 'x86_64'
-
-    - name: Build/Test/Package
-      env:
-        CIBW_ARCHS: all
-        # HACK: ick, maybe deconstruct the matrix a bit or query cibuildwheel for its default target *linux spec first?
-        CIBW_BUILD: ${{matrix.spec}}-${{ contains(matrix.platform, 'musllinux') && 'musllinux' || 'manylinux' }}_${{matrix.arch}}
-        CIBW_BUILD_VERBOSITY: 1
-        # containerized Linux builds require explicit CIBW_ENVIRONMENT
-        CIBW_ENVIRONMENT: >
-          C_INCLUDE_PATH=libyaml/include
-          LIBRARY_PATH=libyaml/src/.libs
-          LD_LIBRARY_PATH=libyaml/src/.libs
-          PYYAML_FORCE_CYTHON=1
-          PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; pytest
-        CIBW_TEST_REQUIRES: pytest
-      run: |
-        set -eux
-        python3 -V
-        python3 -m pip install -U --user cibuildwheel
-        python3 -m cibuildwheel --platform auto --output-dir dist .
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: dist
-        path: dist/*.whl
-        if-no-files-found: error
-
-  macos_libyaml:
-    name: libyaml macos ${{matrix.arch}}
-    strategy:
-      matrix:
-        include:
-        - arch: x86_64
-        - arch: arm64
-          runs_on: [self-hosted, macOS, arm64]
-          deployment_target: '11.0'
-          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-          sdkroot: macosx11.3
-    defaults:
-      run:
-        shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
-    runs-on: ${{ matrix.runs_on || 'macos-11' }}
-    steps:
-    - name: Check cached libyaml state
-      id: cached_libyaml
-      uses: actions/cache@v3
-      with:
-        path: libyaml
-        key: libyaml_macos_${{matrix.arch}}_${{env.LIBYAML_REF}}
-
-    - name: Checkout PyYAML
-      uses: actions/checkout@v3
-      if: steps.cached_libyaml.outputs.cache-hit != 'true'
-
-    - name: Build libyaml
-      env:
-        MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
-        SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
-      run: |
-        set -eux
-        brew install automake coreutils m4
-        bash ./packaging/build/libyaml.sh
-        echo "finished artifact arch is $(lipo -archs libyaml/src/.libs/libyaml.a)"
-      if: steps.cached_libyaml.outputs.cache-hit != 'true'
-
-
-  macos_pyyaml:
-    needs: macos_libyaml
-    name: pyyaml ${{ matrix.spec }}
-    runs-on: ${{ matrix.runs_on || 'macos-11' }}
-    defaults:
-      run:
-        shell: ${{ matrix.run_wrapper || 'bash --noprofile --norc -eo pipefail {0}' }}
-    strategy:
-      matrix:
-        include:
-        - spec: cp38-macosx_x86_64
-          cibw_version: cibuildwheel==2.11.1
-        - spec: cp39-macosx_x86_64
-        - spec: cp310-macosx_x86_64
-        - spec: cp311-macosx_x86_64
-        - spec: cp312-macosx_x86_64
-
-        # build for arm64 under a hacked macOS 12 self-hosted x86_64-on-arm64 runner until arm64 is fully supported
-        - spec: cp39-macosx_arm64
-          deployment_target: '11.0'
-          runs_on: [self-hosted, macOS, arm64]
-          arch: arm64
-          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-          sdkroot: macosx11.3
-
-        - spec: cp310-macosx_arm64
-          deployment_target: '11.0'
-          runs_on: [self-hosted, macOS, arm64]
-          arch: arm64
-          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-          sdkroot: macosx11.3
-
-        - spec: cp311-macosx_arm64
-          deployment_target: '11.0'
-          runs_on: [self-hosted, macOS, arm64]
-          arch: arm64
-          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-          sdkroot: macosx11.3
-
-        - spec: cp312-macosx_arm64
-          deployment_target: '11.0'
-          runs_on: [self-hosted, macOS, arm64]
-          arch: arm64
-          run_wrapper: arch -arm64 bash --noprofile --norc -eo pipefail {0}
-          sdkroot: macosx11.3
-
-    steps:
-    - name: Checkout PyYAML
-      uses: actions/checkout@v3
-
-    - name: Get cached libyaml state
-      id: cached_libyaml
-      uses: actions/cache/restore@v3
-      with:
-        path: libyaml
-        key: libyaml_macos_${{ matrix.arch || 'x86_64' }}_${{env.LIBYAML_REF}}
-        fail-on-cache-miss: true
-
-    - name: Build/Test/Package
-      env:
-        C_INCLUDE_PATH: libyaml/include
-        CIBW_BUILD: ${{matrix.spec}}
-        CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; pytest
-        CIBW_TEST_REQUIRES: pytest
-        LIBRARY_PATH: libyaml/src/.libs
-        MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
-        PYYAML_FORCE_CYTHON: 1
-        PYYAML_FORCE_LIBYAML: 1
-        SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
-      run: |
-        python3 -V
-        python3 -m pip install -U --user ${{ matrix.cibw_version || 'cibuildwheel' }}
-        python3 -m cibuildwheel --platform auto --output-dir dist .
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: dist
-        path: dist/*.whl
-        if-no-files-found: error
-
-  windows_libyaml:
-    name: libyaml ${{matrix.platform}} ${{matrix.arch}}
-    runs-on: ${{matrix.platform}}
-    strategy:
-      matrix:
-        include:
-        - platform: windows-2019
-          arch: x64
-        - platform: windows-2019
-          arch: win32
-    steps:
-    - name: Get cached libyaml state
-      id: cached_libyaml
-      uses: actions/cache@v3
-      with:
-        path: libyaml
-        key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}}
-
-    - name: Build libyaml
-      shell: bash
-      if: steps.cached_libyaml.outputs.cache-hit != 'true'
-      run: |
-        # git spews all over stderr unless we tell it not to
-        export GIT_REDIRECT_STDERR="2>&1"
-
-        if [[ ! -d ./libyaml ]]; then
-          git clone -b ${{ env.LIBYAML_REF }} ${{ env.LIBYAML_REPO }} 2>&1
-        fi
-
-        pushd libyaml
-        git clean -fdx
-        popd
-
-        mkdir libyaml/build
-
-        pushd libyaml/build
-        cmake.exe -G "Visual Studio 16 2019" -A ${{ matrix.arch }} -DYAML_STATIC_LIB_NAME=yaml ..
-        cmake.exe --build . --config Release
-        popd
-
-
-  windows_pyyaml:
-    needs: windows_libyaml
-    name: pyyaml ${{ matrix.platform }} ${{matrix.python_arch}} ${{matrix.spec}}
-    runs-on: ${{matrix.platform}}
-    strategy:
-      matrix:
-        include:
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: 3.8
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: 3.9
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: '3.10'
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: '3.11'
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: '3.12'
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: 3.8
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: 3.9
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: '3.10'
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: '3.11'
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: '3.12'
-    steps:
-    # autocrlf screws up tests under Windows
-    - name: Set git to use LF
-      run: |
-        git config --global core.autocrlf false
-        git config --global core.eol lf
-
-    - name: Checkout pyyaml
-      uses: actions/checkout@v3
-
-    - name: Get cached libyaml state
-      id: cached_libyaml
-      uses: actions/cache/restore@v3
-      with:
-        path: libyaml
-        key: libyaml_${{matrix.platform}}_${{matrix.build_arch}}_${{env.LIBYAML_REF}}
-        fail-on-cache-miss: true
-
-    - name: Install python ${{ matrix.spec }}
-      uses: actions/setup-python@v4
-      with:
-        architecture: ${{ matrix.python_arch }}
-        python-version: ${{ matrix.spec }}
-
-    - name: Build/Test/Package
-      env:
-        PYYAML_FORCE_CYTHON: 1
-        PYYAML_FORCE_LIBYAML: 1
-      shell: bash
-      run: |
-        set -eux
-        python -V
-        python -m pip install "Cython<3.0" setuptools wheel
-
-        python setup.py \
-        --with-libyaml build_ext \
-        -I libyaml/include \
-        -L libyaml/build/Release \
-        -D YAML_DECLARE_STATIC \
-        build bdist_wheel
-
-        # run tests on built wheel
-        python -m pip install dist/*.whl pytest
-        python -I -m pytest
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: dist
-        path: dist/*.whl
-        if-no-files-found: error
-...

MANIFEST.in

@@ -1,6 +1,7 @@
 include CHANGES README LICENSE Makefile pyproject.toml setup.py
 recursive-include lib/yaml *.py
 recursive-include lib/_yaml *.py
+recursive-include packaging *
 recursive-include examples *.py *.cfg *.yaml
 recursive-include tests/legacy_tests/ *.py
 recursive-include tests/legacy_tests/data *

packaging/_pyyaml_pep517.py

@@ -0,0 +1,51 @@
+import inspect
+
+
+def _bridge_build_meta():
+    import functools
+    import sys
+
+    from setuptools import build_meta
+
+    self_module = sys.modules[__name__]
+
+    for attr_name in build_meta.__all__:
+        attr_value = getattr(build_meta, attr_name)
+        if callable(attr_value):
+            setattr(self_module, attr_name, functools.partial(_expose_config_settings, attr_value))
+
+
+class ActiveConfigSettings:
+    _current = {}
+
+    def __init__(self, config_settings):
+        self._config = config_settings
+
+    def __enter__(self):
+        type(self)._current = self._config
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        type(self)._current = {}
+
+    @classmethod
+    def current(cls):
+        return cls._current
+
+
+def _expose_config_settings(real_method, *args, **kwargs):
+    from contextlib import nullcontext
+    import inspect
+
+    sig = inspect.signature(real_method)
+    boundargs = sig.bind(*args, **kwargs)
+
+    config = boundargs.arguments.get('config_settings')
+
+    ctx = ActiveConfigSettings(config) if config else nullcontext()
+
+    with ctx:
+        return real_method(*args, **kwargs)
+
+
+_bridge_build_meta()
+

pyproject.toml

@@ -1,3 +1,9 @@
 [build-system]
-requires = ["setuptools", "wheel", "Cython<3.0"]
+requires = [
-build-backend = "setuptools.build_meta"
+    "setuptools",  # FIXME: declare min/max setuptools versions?
+    "wheel",
+    "Cython; python_version < '3.13'",
+    "Cython>=3.0; python_version >= '3.13'"
+]
+backend-path = ["packaging"]
+build-backend = "_pyyaml_pep517"

setup.py

@@ -34,6 +34,8 @@ CLASSIFIERS = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Software Development :: Libraries :: Python Modules",
@@ -82,7 +84,12 @@ if 'sdist' in sys.argv or os.environ.get('PYYAML_FORCE_CYTHON') == '1':
     with_cython = True
 try:
     from Cython.Distutils.extension import Extension as _Extension
-    from Cython.Distutils import build_ext as _build_ext
+    try:
+        # try old_build_ext from Cython > 3 first, until we can dump it entirely
+        from Cython.Distutils.old_build_ext import old_build_ext as _build_ext
+    except ImportError:
+        # Cython < 3
+        from Cython.Distutils import build_ext as _build_ext
     with_cython = True
 except ImportError:
     if with_cython:
@@ -94,6 +101,14 @@ except ImportError:
     bdist_wheel = None
 
 
+try:
+    from _pyyaml_pep517 import ActiveConfigSettings
+except ImportError:
+    class ActiveConfigSettings:
+        @staticmethod
+        def current():
+            return {}
+
 # on Windows, disable wheel generation warning noise
 windows_ignore_warnings = [
 "Unknown distribution option: 'python_requires'",
@@ -173,6 +188,31 @@ class Extension(_Extension):
 
 
 class build_ext(_build_ext):
+    def finalize_options(self):
+        super().finalize_options()
+        pep517_config = ActiveConfigSettings.current()
+
+        build_config = pep517_config.get('pyyaml_build_config')
+
+        if build_config:
+            import json
+            build_config = json.loads(build_config)
+            print(f"`pyyaml_build_config`: {build_config}")
+        else:
+            build_config = {}
+            print("No `pyyaml_build_config` setting found.")
+
+        for key, value in build_config.items():
+            existing_value = getattr(self, key, ...)
+            if existing_value is ...:
+                print(f"ignoring unknown config key {key!r}")
+                continue
+
+            if existing_value:
+                print(f"combining {key!r} {existing_value!r} and {value!r}")
+                value = existing_value + value  # FIXME: handle type diff
+
+            setattr(self, key, value)
 
     def run(self):
         optional = True
@@ -230,6 +270,7 @@ class build_ext(_build_ext):
             if with_ext is not None and not with_ext:
                 continue
             if with_cython:
+                print(f"BUILDING CYTHON EXT; {self.include_dirs=} {self.library_dirs=} {self.define=}")
                 ext.sources = self.cython_sources(ext.sources, ext)
             try:
                 self.build_extension(ext)

tests/legacy_tests/conftest.py

@@ -2,6 +2,7 @@
 # legacy test case as a pyyaml item
 
 import os
+import pathlib
 import pytest
 import warnings
 
@@ -111,8 +112,9 @@ def pytest_pycollect_makeitem(collector, name: str, obj: object):
 def pytest_collection_modifyitems(session, config, items):
     pass
 
-def pytest_ignore_collect(path, config):
+
-    basename = path.basename
+def pytest_ignore_collect(collection_path: pathlib.Path):
+    basename = collection_path.name
     # ignore all Python files in this subtree for normal pytest collection
     if basename not in ['test_yaml.py', 'test_yaml_ext.py']:
         return True

tests/legacy_tests/test_appliance.py

@@ -1,7 +1,8 @@
 
-import sys, os, os.path, types, traceback, pprint
+import sys, os, os.path, types, traceback, pprint, pathlib
+
+DATA = str(pathlib.Path(__file__).parent / 'data')
 
-DATA = 'tests/legacy_tests/data'
 
 def find_test_functions(collections):
     if not isinstance(collections, list):