Python 3.14.1

Fix a potential virtual memory allocation denial of service in http.server (GH-119455)"

.devcontainer/devcontainer.json

@@ -1,13 +1,10 @@
 {
-    "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730",
+    "image": "ghcr.io/python/devcontainer:latest",
     "onCreateCommand": [
         // Install common tooling.
         "dnf",
         "install",
         "-y",
-        "which",
-        "zsh",
-        "fish",
         // For umask fix below.
         "/usr/bin/setfacl"
     ],

.editorconfig

@@ -1,11 +1,11 @@
 root = true
 
-[*.{py,c,cpp,h,js,rst,md,yml,yaml}]
+[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}]
 trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = space
 
-[*.{py,c,cpp,h}]
+[*.{py,c,cpp,h,gram}]
 indent_size = 4
 
 [*.rst]

.gitattributes

@@ -10,6 +10,7 @@
 *.ico binary
 *.jpg binary
 *.pck binary
+*.pdf binary
 *.png binary
 *.psd binary
 *.tar binary
@@ -67,6 +68,7 @@ PCbuild/readme.txt  dos
 **/clinic/*.cpp.h                                   generated
 **/clinic/*.h.h                                     generated
 *_db.h                                              generated
+Doc/c-api/lifecycle.dot.svg                         generated
 Doc/data/stable_abi.dat                             generated
 Doc/library/token-list.inc                          generated
 Include/internal/pycore_ast.h                       generated
@@ -81,6 +83,7 @@ Include/opcode_ids.h                                generated
 Include/token.h                                     generated
 Lib/_opcode_metadata.py                             generated
 Lib/keyword.py                                      generated
+Lib/idlelib/help.html                               generated
 Lib/test/certdata/*.pem                             generated
 Lib/test/certdata/*.0                               generated
 Lib/test/levenshtein_examples.json                  generated

.github/CODEOWNERS

@@ -8,7 +8,7 @@
 .github/**                    @ezio-melotti @hugovk @AA-Turner
 
 # pre-commit
-.pre-commit-config.yaml       @hugovk @AlexWaygood
+.pre-commit-config.yaml       @hugovk
 .ruff.toml                    @hugovk @AlexWaygood @AA-Turner
 
 # Build system
@@ -16,6 +16,10 @@ configure*                    @erlend-aasland @corona10
 Makefile.pre.in               @erlend-aasland
 Modules/Setup*                @erlend-aasland
 
+# generate-build-details
+Tools/build/generate-build-details.py   @FFY00
+Lib/test/test_build_details.py          @FFY00
+
 # argparse
 **/*argparse*                 @savannahostrowski
 
@@ -27,6 +31,7 @@ Modules/Setup*                @erlend-aasland
 **/*genobject*                @markshannon
 **/*hamt*                     @1st1
 **/*jit*                      @brandtbucher @savannahostrowski
+Python/perf_jit_trampoline.c  # Exclude the owners of "**/*jit*", above.
 Objects/set*                  @rhettinger
 Objects/dict*                 @methane @markshannon
 Objects/typevarobject.c       @JelleZijlstra
@@ -281,14 +286,21 @@ Doc/howto/clinic.rst          @erlend-aasland
 # Subinterpreters
 **/*interpreteridobject.*     @ericsnowcurrently
 **/*crossinterp*              @ericsnowcurrently
-Lib/test/support/interpreters/  @ericsnowcurrently
 Modules/_interp*module.c      @ericsnowcurrently
+Lib/test/test__interp*.py     @ericsnowcurrently
+Lib/concurrent/interpreters/  @ericsnowcurrently
+Lib/test/support/channels.py  @ericsnowcurrently
+Doc/library/concurrent.interpreters.rst  @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently
+Lib/concurrent/futures/interpreter.py  @ericsnowcurrently
 
 # Android
 **/*Android*                  @mhsmith @freakboy3742
 **/*android*                  @mhsmith @freakboy3742
 
+# Apple
+/Apple                        @freakboy3742
+
 # iOS (but not termios)
 **/iOS*                       @freakboy3742
 **/ios*                       @freakboy3742
@@ -298,7 +310,12 @@ Lib/test/test_interpreters/   @ericsnowcurrently
 **/*-ios*                     @freakboy3742
 
 # WebAssembly
-/Tools/wasm/                  @brettcannon @freakboy3742
+Tools/wasm/config.site-wasm32-emscripten  @freakboy3742
+/Tools/wasm/README.md         @brettcannon @freakboy3742
+/Tools/wasm/wasi-env          @brettcannon
+/Tools/wasm/wasi_build.py     @brettcannon
+/Tools/wasm/emscripten        @freakboy3742
+/Tools/wasm/wasi              @brettcannon
 
 # SBOM
 /Misc/externals.spdx.json     @sethmlarson
@@ -326,3 +343,8 @@ Modules/_xxtestfuzz/          @ammaraskar
 **/*templateobject*           @lysnikolaou
 **/*templatelib*              @lysnikolaou
 **/*tstring*                  @lysnikolaou
+
+# Remote debugging
+Python/remote_debug.h         @pablogsal
+Python/remote_debugging.c     @pablogsal
+Modules/_remote_debugging_module.c @pablogsal @ambv @1st1

.github/actionlint.yaml

@@ -1,6 +1,7 @@
 self-hosted-runner:
   # Pending https://github.com/rhysd/actionlint/issues/533
-  labels: ["windows-11-arm"]
+  # and https://github.com/rhysd/actionlint/issues/571
+  labels: ["windows-11-arm", "macos-15-intel"]
 
 config-variables: null
 

.github/workflows/build.yml

@@ -15,7 +15,13 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable
+  # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency
+  # 'group' must be a key uniquely representing a PR or push event.
+  # github.workflow is the workflow name
+  # github.actor is the user invoking the workflow
+  # github.head_ref is the source branch of the PR or otherwise blank
+  # github.run_id is a unique number for the current run
+  group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 env:
@@ -43,6 +49,53 @@ jobs:
     if: fromJSON(needs.build-context.outputs.run-docs)
     uses: ./.github/workflows/reusable-docs.yml
 
+  check-abi:
+    name: 'Check if the ABI has changed'
+    runs-on: ubuntu-22.04  # 24.04 causes spurious errors
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+            sudo ./.github/workflows/posix-deps-apt.sh
+            sudo apt-get install -yq abigail-tools
+      - name: Build CPython
+        env:
+          CFLAGS: -g3 -O0
+        run: |
+          # Build Python with the libpython dynamic library
+          ./configure --enable-shared
+          make -j4
+      - name: Check for changes in the ABI
+        id: check
+        run: |
+          if ! make check-abidump; then
+            echo "Generated ABI file is not up to date."
+            echo "Please add the release manager of this branch as a reviewer of this PR."
+            echo ""
+            echo "The up to date ABI file should be attached to this build as an artifact."
+            echo ""
+            echo "To learn more about this check: https://devguide.python.org/getting-started/setup-building/index.html#regenerate-the-abi-dump"
+            echo ""
+            exit 1
+          fi
+      - name: Generate updated ABI files
+        if: ${{ failure() && steps.check.conclusion == 'failure' }}
+        run: |
+          make regen-abidump
+      - uses: actions/upload-artifact@v4
+        name: Publish updated ABI files
+        if: ${{ failure() && steps.check.conclusion == 'failure' }}
+        with:
+          name: abi-data
+          path: ./Doc/data/*.abi
+
   check-autoconf-regen:
     name: 'Check if Autoconf files are up to date'
     # Don't use ubuntu-latest but a specific version to make the job
@@ -103,20 +156,10 @@ jobs:
           python-version: '3.x'
       - name: Runner image version
         run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-      - name: Restore config.cache
-        uses: actions/cache@v4
-        with:
-          path: config.cache
-          # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
-          key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }}
       - name: Install dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
         run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
-      - name: Configure ccache action
-        uses: hendrikmuhs/ccache-action@v1.2
-        with:
-          save: false
       - name: Configure CPython
         run: |
           # Build Python with the libpython dynamic library
@@ -124,7 +167,7 @@ jobs:
       - name: Build CPython
         run: |
           make -j4 regen-all
-          make regen-stdlib-module-names regen-sbom regen-unicodedata
+          make regen-stdlib-module-names regen-sbom
       - name: Check for changes
         run: |
           git add -u
@@ -172,8 +215,8 @@ jobs:
       free-threading: ${{ matrix.free-threading }}
 
   build-windows-msi:
-    name: >-  # ${{ '' } is a hack to nest jobs under the same sidebar category
-      Windows MSI${{ '' }}
+    # ${{ '' } is a hack to nest jobs under the same sidebar category.
+    name: Windows MSI${{ '' }}  # zizmor: ignore[obfuscation]
     needs: build-context
     if: fromJSON(needs.build-context.outputs.run-windows-msi)
     strategy:
@@ -196,28 +239,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Cirrus and macos-14 are M1, macos-13 is default GHA Intel.
-        # macOS 13 only runs tests against the GIL-enabled CPython.
-        # Cirrus used for upstream, macos-14 for forks.
+        # macos-14 is M1, macos-15-intel is Intel.
+        # macos-15-intel only runs tests against the GIL-enabled CPython.
         os:
-        - ghcr.io/cirruslabs/macos-runner:sonoma
         - macos-14
-        - macos-13
-        is-fork:  # only used for the exclusion trick
-        - ${{ github.repository_owner != 'python' }}
+        - macos-15-intel
         free-threading:
         - false
         - true
         exclude:
-        - os: ghcr.io/cirruslabs/macos-runner:sonoma
-          is-fork: true
-        - os: macos-14
-          is-fork: false
-        - os: macos-13
+        - os: macos-15-intel
           free-threading: true
     uses: ./.github/workflows/reusable-macos.yml
     with:
-      config_hash: ${{ needs.build-context.outputs.config-hash }}
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
 
@@ -249,7 +283,6 @@ jobs:
           bolt: true
     uses: ./.github/workflows/reusable-ubuntu.yml
     with:
-      config_hash: ${{ needs.build-context.outputs.config-hash }}
       bolt-optimizations: ${{ matrix.bolt }}
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
@@ -264,7 +297,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04]
-        openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1]
+        # Keep 1.1.1w in our list despite it being upstream EOL and otherwise
+        # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
+        # supported by important vendors such as AWS-LC.
+        openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4]
         # See Tools/ssl/make_ssl_data.py for notes on adding a new version
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
@@ -277,11 +313,6 @@ jobs:
         persist-credentials: false
     - name: Runner image version
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: Restore config.cache
-      uses: actions/cache@v4
-      with:
-        path: config.cache
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install dependencies
@@ -303,10 +334,6 @@ jobs:
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
-    - name: Configure ccache action
-      uses: hendrikmuhs/ccache-action@v1.2
-      with:
-        save: false
     - name: Configure CPython
       run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR"
     - name: Build CPython
@@ -316,13 +343,56 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  build-android:
+    name: Android (${{ matrix.arch }})
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: aarch64
+            runs-on: macos-14
+          - arch: x86_64
+            runs-on: ubuntu-24.04
+
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Build and test
+        run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android
+
+  build-ios:
+    name: iOS
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
+    timeout-minutes: 60
+    runs-on: macos-15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      # GitHub recommends explicitly selecting the desired Xcode version:
+      # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140
+      # This became a necessity as a result of
+      # https://github.com/actions/runner-images/issues/12541 and
+      # https://github.com/actions/runner-images/issues/12751.
+      - name: Select Xcode version
+        run: |
+          sudo xcode-select --switch /Applications/Xcode_16.4.app
+
+      - name: Build and test
+        run: python3 Apple ci iOS --fast-ci --simulator 'iPhone 16e,OS=18.5'
+
   build-wasi:
     name: 'WASI'
     needs: build-context
     if: needs.build-context.outputs.run-tests == 'true'
     uses: ./.github/workflows/reusable-wasi.yml
-    with:
-      config_hash: ${{ needs.build-context.outputs.config-hash }}
 
   test-hypothesis:
     name: "Hypothesis tests on Ubuntu"
@@ -331,7 +401,7 @@ jobs:
     needs: build-context
     if: needs.build-context.outputs.run-tests == 'true'
     env:
-      OPENSSL_VER: 3.0.16
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -358,10 +428,6 @@ jobs:
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
-    - name: Configure ccache action
-      uses: hendrikmuhs/ccache-action@v1.2
-      with:
-        save: false
     - name: Setup directory envs for out-of-tree builds
       run: |
         echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV"
@@ -372,11 +438,6 @@ jobs:
       run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR"
     - name: Runner image version
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: Restore config.cache
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -451,7 +512,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
     env:
-      OPENSSL_VER: 3.0.16
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
@@ -460,11 +521,6 @@ jobs:
         persist-credentials: false
     - name: Runner image version
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: Restore config.cache
-      uses: actions/cache@v4
-      with:
-        path: config.cache
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install dependencies
@@ -490,11 +546,6 @@ jobs:
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
-    - name: Configure ccache action
-      uses: hendrikmuhs/ccache-action@v1.2
-      with:
-        save: ${{ github.event_name == 'push' }}
-        max-size: "200M"
     - name: Configure CPython
       run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
     - name: Build CPython
@@ -504,21 +555,28 @@ jobs:
     - name: Tests
       run: xvfb-run make ci
 
-  build-tsan:
-    name: >-
-      Thread sanitizer
-      ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
+  build-san:
+    # ${{ '' } is a hack to nest jobs under the same sidebar category.
+    name: Sanitizers${{ '' }}  # zizmor: ignore[obfuscation]
     needs: build-context
     if: needs.build-context.outputs.run-tests == 'true'
     strategy:
       fail-fast: false
       matrix:
+        check-name:
+        - Thread
         free-threading:
         - false
         - true
-    uses: ./.github/workflows/reusable-tsan.yml
+        sanitizer:
+        - TSan
+        include:
+        - check-name: Undefined behavior
+          sanitizer: UBSan
+          free-threading: false
+    uses: ./.github/workflows/reusable-san.yml
     with:
-      config_hash: ${{ needs.build-context.outputs.config-hash }}
+      sanitizer: ${{ matrix.sanitizer }}
       free-threading: ${{ matrix.free-threading }}
 
   cross-build-linux:
@@ -533,11 +591,6 @@ jobs:
           persist-credentials: false
       - name: Runner image version
         run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-      - name: Restore config.cache
-        uses: actions/cache@v4
-        with:
-          path: config.cache
-          key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
       - name: Register gcc problem matcher
         run: echo "::add-matcher::.github/problem-matchers/gcc.json"
       - name: Set build dir
@@ -615,10 +668,12 @@ jobs:
     - build-macos
     - build-ubuntu
     - build-ubuntu-ssltests
+    - build-android
+    - build-ios
     - build-wasi
     - test-hypothesis
     - build-asan
-    - build-tsan
+    - build-san
     - cross-build-linux
     - cifuzz
     if: always()
@@ -648,10 +703,12 @@ jobs:
             build-macos,
             build-ubuntu,
             build-ubuntu-ssltests,
+            build-android,
+            build-ios,
             build-wasi,
             test-hypothesis,
             build-asan,
-            build-tsan,
+            build-san,
             cross-build-linux,
             '
             || ''

.github/workflows/jit.yml

@@ -68,16 +68,16 @@ jobs:
         include:
           - target: i686-pc-windows-msvc/msvc
             architecture: Win32
-            runner: windows-latest
+            runner: windows-2022
           - target: x86_64-pc-windows-msvc/msvc
             architecture: x64
-            runner: windows-latest
+            runner: windows-2022
           - target: aarch64-pc-windows-msvc/msvc
             architecture: ARM64
             runner: windows-11-arm
           - target: x86_64-apple-darwin/clang
             architecture: x86_64
-            runner: macos-13
+            runner: macos-15-intel
           - target: aarch64-apple-darwin/clang
             architecture: aarch64
             runner: macos-14
@@ -102,17 +102,16 @@ jobs:
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-        # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
-        # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
-        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
-        # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
       - name: macOS
         if: runner.os == 'macOS'
         run: |
           brew update
-          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
           brew install llvm@${{ matrix.llvm }}
           export SDKROOT="$(xcrun --show-sdk-path)"
+          # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to
+          # make sure we don't break downstream distributors (like uv):
+          export CFLAGS_JIT='-Werror=unguarded-availability'
+          export MACOSX_DEPLOYMENT_TARGET=10.15
           ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }}
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
@@ -153,3 +152,26 @@ jobs:
   #     - name: Run tests
   #       run: |
   #         ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+  tail-call-jit:
+    name: JIT with tail calling interpreter
+    needs: interpreter
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        llvm:
+          - 19
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Build with JIT and tailcall
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug
+          make all --jobs 4

.github/workflows/mypy.yml

@@ -13,13 +13,22 @@ on:
       - "Lib/test/libregrtest/**"
       - "Lib/tomllib/**"
       - "Misc/mypy/**"
+      - "Tools/build/check_extension_modules.py"
+      - "Tools/build/check_warnings.py"
+      - "Tools/build/compute-changes.py"
+      - "Tools/build/deepfreeze.py"
+      - "Tools/build/generate-build-details.py"
       - "Tools/build/generate_sbom.py"
+      - "Tools/build/generate_stdlib_module_names.py"
+      - "Tools/build/mypy.ini"
+      - "Tools/build/umarshal.py"
+      - "Tools/build/update_file.py"
+      - "Tools/build/verify_ensurepip_wheels.py"
       - "Tools/cases_generator/**"
       - "Tools/clinic/**"
       - "Tools/jit/**"
       - "Tools/peg_generator/**"
       - "Tools/requirements-dev.txt"
-      - "Tools/wasm/**"
   workflow_dispatch:
 
 permissions:
@@ -51,7 +60,6 @@ jobs:
           "Tools/clinic",
           "Tools/jit",
           "Tools/peg_generator",
-          "Tools/wasm",
         ]
     steps:
       - uses: actions/checkout@v4

.github/workflows/posix-deps-apt.sh

@@ -17,6 +17,7 @@ apt-get -yq install \
     libreadline6-dev \
     libsqlite3-dev \
     libssl-dev \
+    libzstd-dev \
     lzma \
     lzma-dev \
     strace \

.github/workflows/reusable-context.yml

@@ -17,9 +17,6 @@ on:  # yamllint disable-line rule:truthy
       #        || 'falsy-branch'
       #   }}
       #
-      config-hash:
-        description: Config hash value for use in cache keys
-        value: ${{ jobs.compute-changes.outputs.config-hash }}  # str
       run-docs:
         description: Whether to build the docs
         value: ${{ jobs.compute-changes.outputs.run-docs }}  # bool
@@ -42,7 +39,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
-      config-hash: ${{ steps.config-hash.outputs.hash }}
       run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
       run-docs: ${{ steps.changes.outputs.run-docs }}
       run-tests: ${{ steps.changes.outputs.run-tests }}
@@ -97,10 +93,6 @@ jobs:
       run: python Tools/build/compute-changes.py
       env:
         GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
         CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }}
         CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }}
-
-    - name: Compute hash for config cache key
-      id: config-hash
-      run: |
-        echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT"

.github/workflows/reusable-docs.yml

@@ -66,7 +66,7 @@ jobs:
       run: |
         set -Eeuo pipefail
         # Build docs with the nit-picky option; write warnings to file
-        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html
+        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html
     - name: 'Check warnings'
       if: github.event_name == 'pull_request'
       run: |
@@ -102,3 +102,30 @@ jobs:
     # Use "xvfb-run" since some doctest tests open GUI windows
     - name: 'Run documentation doctest'
       run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest
+
+  check-epub:
+    name: 'Check EPUB'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: 'Set up Python'
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3'
+        cache: 'pip'
+        cache-dependency-path: 'Doc/requirements.txt'
+    - name: 'Install build dependencies'
+      run: |
+        make -C Doc/ venv
+        python -m pip install epubcheck
+    - name: 'Build EPUB documentation'
+      run: make -C Doc/ PYTHON=../python epub
+    - name: 'Run epubcheck'
+      continue-on-error: true
+      run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt
+    - run: cat Doc/epubcheck.txt
+    - name: 'Check for fatal errors in EPUB'
+      run: python Doc/tools/check-epub.py

.github/workflows/reusable-macos.yml

@@ -3,9 +3,6 @@ name: Reusable macOS
 on:
   workflow_call:
     inputs:
-      config_hash:
-        required: true
-        type: string
       free-threading:
         required: false
         type: boolean
@@ -36,11 +33,6 @@ jobs:
         persist-credentials: false
     - name: Runner image version
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: Restore config.cache
-      uses: actions/cache@v4
-      with:
-        path: config.cache
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
       run: |
         brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make
@@ -60,15 +52,15 @@ jobs:
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix openssl@3.0)"
     - name: Build CPython
-      if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+      if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }}
       run: gmake -j8
     - name: Build CPython for compiler warning check
-      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }}
       run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
     - name: Display build info
       run: make pythoninfo
     - name: Check compiler warnings
-      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }}
       run: >-
         python3 Tools/build/check_warnings.py
         --compiler-output-file-path=compiler_output_macos.txt

.github/workflows/reusable-san.yml

@@ -0,0 +1,111 @@
+name: Reusable Sanitizer
+
+on:
+  workflow_call:
+    inputs:
+      sanitizer:
+        required: true
+        type: string
+      free-threading:
+        description: Whether to use free-threaded mode
+        required: false
+        type: boolean
+        default: false
+
+env:
+  FORCE_COLOR: 1
+
+jobs:
+  build-san-reusable:
+    name: >-
+      ${{ inputs.sanitizer }}${{
+        inputs.free-threading
+        && ' (free-threading)'
+        || ''
+      }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Runner image version
+      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
+    - name: Install dependencies
+      run: |
+        sudo ./.github/workflows/posix-deps-apt.sh
+        # Install clang
+        wget https://apt.llvm.org/llvm.sh
+        chmod +x llvm.sh
+
+        if [ "${SANITIZER}" = "TSan" ]; then
+          sudo ./llvm.sh 17  # gh-121946: llvm-18 package is temporarily broken
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100
+          sudo update-alternatives --set clang /usr/bin/clang-17
+          sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100
+          sudo update-alternatives --set clang++ /usr/bin/clang++-17
+          # Reduce ASLR to avoid TSan crashing
+          sudo sysctl -w vm.mmap_rnd_bits=28
+        else
+          sudo ./llvm.sh 20
+        fi
+
+    - name: Sanitizer option setup
+      run: |
+        if [ "${SANITIZER}" = "TSan" ]; then
+          echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{
+              fromJSON(inputs.free-threading)
+              && '_free_threading'
+              || ''
+            }}.txt handle_segv=0" >> "$GITHUB_ENV"
+        else
+          echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV"
+        fi
+        echo "CC=clang" >> "$GITHUB_ENV"
+        echo "CXX=clang++" >> "$GITHUB_ENV"
+      env:
+        SANITIZER: ${{ inputs.sanitizer }}
+        SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
+    - name: Configure CPython
+      run: >-
+        ./configure
+        --config-cache
+        ${{
+          inputs.sanitizer == 'TSan'
+          && '--with-thread-sanitizer'
+          || '--with-undefined-behavior-sanitizer'
+        }}
+        --with-pydebug
+        ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: >-
+        ./python -m test
+        ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }}
+        -j4
+    - name: Parallel tests
+      if: >-
+        inputs.sanitizer == 'TSan'
+        && fromJSON(inputs.free-threading)
+      run: ./python -m test --tsan-parallel --parallel-threads=4 -j4
+    - name: Display logs
+      if: always()
+      run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: >-
+          ${{ inputs.sanitizer }}-logs-${{
+            fromJSON(inputs.free-threading)
+            && 'free-threading'
+            || 'default'
+          }}
+        path: san_log.*
+        if-no-files-found: ignore

.github/workflows/reusable-tsan.yml

@@ -1,94 +0,0 @@
-name: Reusable Thread Sanitizer
-
-on:
-  workflow_call:
-    inputs:
-      config_hash:
-        required: true
-        type: string
-      free-threading:
-        description: Whether to use free-threaded mode
-        required: false
-        type: boolean
-        default: false
-
-env:
-  FORCE_COLOR: 1
-
-jobs:
-  build-tsan-reusable:
-    name: 'Thread sanitizer'
-    runs-on: ubuntu-24.04
-    timeout-minutes: 60
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        persist-credentials: false
-    - name: Runner image version
-      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: Restore config.cache
-      uses: actions/cache@v4
-      with:
-        path: config.cache
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }}
-    - name: Install dependencies
-      run: |
-        sudo ./.github/workflows/posix-deps-apt.sh
-        # Install clang-18
-        wget https://apt.llvm.org/llvm.sh
-        chmod +x llvm.sh
-        sudo ./llvm.sh 17  # gh-121946: llvm-18 package is temporarily broken
-        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100
-        sudo update-alternatives --set clang /usr/bin/clang-17
-        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100
-        sudo update-alternatives --set clang++ /usr/bin/clang++-17
-        # Reduce ASLR to avoid TSAN crashing
-        sudo sysctl -w vm.mmap_rnd_bits=28
-    - name: TSAN option setup
-      run: |
-        echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{
-            fromJSON(inputs.free-threading)
-            && '_free_threading'
-            || ''
-          }}.txt handle_segv=0" >> "$GITHUB_ENV"
-        echo "CC=clang" >> "$GITHUB_ENV"
-        echo "CXX=clang++" >> "$GITHUB_ENV"
-    - name: Add ccache to PATH
-      run: |
-        echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
-    - name: Configure ccache action
-      uses: hendrikmuhs/ccache-action@v1.2
-      with:
-        save: ${{ github.event_name == 'push' }}
-        max-size: "200M"
-    - name: Configure CPython
-      run: >-
-        ./configure
-        --config-cache
-        --with-thread-sanitizer
-        --with-pydebug
-        ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
-    - name: Build CPython
-      run: make -j4
-    - name: Display build info
-      run: make pythoninfo
-    - name: Tests
-      run: ./python -m test --tsan -j4
-    - name: Parallel tests
-      if: fromJSON(inputs.free-threading)
-      run: ./python -m test --tsan-parallel --parallel-threads=4 -j4
-    - name: Display TSAN logs
-      if: always()
-      run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000
-    - name: Archive TSAN logs
-      if: always()
-      uses: actions/upload-artifact@v4
-      with:
-        name: >-
-          tsan-logs-${{
-            fromJSON(inputs.free-threading)
-            && 'free-threading'
-            || 'default'
-          }}
-        path: tsan_log.*
-        if-no-files-found: ignore

.github/workflows/reusable-ubuntu.yml

@@ -3,9 +3,6 @@ name: Reusable Ubuntu
 on:
   workflow_call:
     inputs:
-      config_hash:
-        required: true
-        type: string
       bolt-optimizations:
         description: Whether to enable BOLT optimizations
         required: false
@@ -30,7 +27,7 @@ jobs:
     runs-on: ${{ inputs.os }}
     timeout-minutes: 60
     env:
-      OPENSSL_VER: 3.0.15
+      OPENSSL_VER: 3.0.18
       PYTHONSTRICTEXTENSIONBUILD: 1
       TERM: linux
     steps:
@@ -64,11 +61,6 @@ jobs:
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
-    - name: Configure ccache action
-      uses: hendrikmuhs/ccache-action@v1.2
-      with:
-        save: ${{ github.event_name == 'push' }}
-        max-size: "200M"
     - name: Setup directory envs for out-of-tree builds
       run: |
         echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV"
@@ -79,11 +71,6 @@ jobs:
       run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR"
     - name: Runner image version
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: Restore config.cache
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       # `test_unpickle_module_race` writes to the source directory, which is

.github/workflows/reusable-wasi.yml

@@ -2,10 +2,6 @@ name: Reusable WASI
 
 on:
   workflow_call:
-    inputs:
-      config_hash:
-        required: true
-        type: string
 
 env:
   FORCE_COLOR: 1
@@ -42,11 +38,6 @@ jobs:
         mkdir "${WASI_SDK_PATH}" && \
         curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \
         tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip
-    - name: "Configure ccache action"
-      uses: hendrikmuhs/ccache-action@v1.2
-      with:
-        save: ${{ github.event_name == 'push' }}
-        max-size: "200M"
     - name: "Add ccache to PATH"
       run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
     - name: "Install Python"
@@ -55,29 +46,15 @@ jobs:
         python-version: '3.x'
     - name: "Runner image version"
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
-    - name: "Restore Python build config.cache"
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
-        # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python.
-        # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables.
-        # (Make sure to keep the key in sync with the other config.cache step below.)
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
     - name: "Configure build Python"
-      run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
+      run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug
     - name: "Make build Python"
-      run: python3 Tools/wasm/wasi.py make-build-python
-    - name: "Restore host config.cache"
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.CROSS_BUILD_WASI }}/config.cache
-        # Should be kept in sync with the other config.cache step above.
-        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
+      run: python3 Tools/wasm/wasi make-build-python
     - name: "Configure host"
       # `--with-pydebug` inferred from configure-build-python
-      run: python3 Tools/wasm/wasi.py configure-host -- --config-cache
+      run: python3 Tools/wasm/wasi configure-host -- --config-cache
     - name: "Make host"
-      run: python3 Tools/wasm/wasi.py make-host
+      run: python3 Tools/wasm/wasi make-host
     - name: "Display build info"
       run: make --directory "${CROSS_BUILD_WASI}" pythoninfo
     - name: "Test"

.github/workflows/reusable-windows-msi.yml

@@ -17,7 +17,7 @@ env:
 jobs:
   build:
     name: installer for ${{ inputs.arch }}
-    runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }}
+    runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }}
     timeout-minutes: 60
     env:
       ARCH: ${{ inputs.arch }}

.github/workflows/reusable-windows.yml

@@ -21,7 +21,7 @@ env:
 jobs:
   build:
     name: Build and test (${{ inputs.arch }})
-    runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }}
+    runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }}
     timeout-minutes: 60
     env:
       ARCH: ${{ inputs.arch }}

.github/workflows/tail-call.yml

@@ -49,16 +49,16 @@ jobs:
         include:
 #          - target: i686-pc-windows-msvc/msvc
 #            architecture: Win32
-#            runner: windows-latest
+#            runner: windows-2022
           - target: x86_64-pc-windows-msvc/msvc
             architecture: x64
-            runner: windows-latest
+            runner: windows-2022
 #          - target: aarch64-pc-windows-msvc/msvc
 #            architecture: ARM64
-#            runner: windows-latest
+#            runner: windows-2022
           - target: x86_64-apple-darwin/clang
             architecture: x86_64
-            runner: macos-13
+            runner: macos-15-intel
           - target: aarch64-apple-darwin/clang
             architecture: aarch64
             runner: macos-14
@@ -101,21 +101,14 @@ jobs:
           set LLVMInstallDir=C:\Program Files\LLVM
           ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }}
 
-        # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
-        # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
-        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
-        # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
-        # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail.
-        # We either need to upgrade LLVM or change the directory being pointed to.
       - name: Native macOS (release)
         if: runner.os == 'macOS'
         run: |
           brew update
-          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
           brew install llvm@${{ matrix.llvm }}
           export SDKROOT="$(xcrun --show-sdk-path)"
-          export PATH="/usr/local/opt/llvm/bin:$PATH"
-          export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
+          export PATH="/usr/local/opt/llvm@${{ matrix.llvm }}/bin:$PATH"
+          export PATH="/opt/homebrew/opt/llvm@${{ matrix.llvm }}/bin:$PATH"
           CC=clang-20 ./configure --with-tail-call-interp
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
@@ -137,4 +130,3 @@ jobs:
           CC=clang-20 ./configure --with-tail-call-interp --disable-gil
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
-

.gitignore

@@ -71,16 +71,15 @@ Lib/test/data/*
 /Makefile
 /Makefile.pre
 /iOSTestbed.*
-iOS/Frameworks/
-iOS/Resources/Info.plist
-iOS/testbed/build
-iOS/testbed/Python.xcframework/ios-*/bin
-iOS/testbed/Python.xcframework/ios-*/include
-iOS/testbed/Python.xcframework/ios-*/lib
-iOS/testbed/Python.xcframework/ios-*/Python.framework
-iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace
-iOS/testbed/iOSTestbed.xcodeproj/xcuserdata
-iOS/testbed/iOSTestbed.xcodeproj/xcshareddata
+Apple/iOS/Frameworks/
+Apple/iOS/Resources/Info.plist
+Apple/testbed/build
+Apple/testbed/Python.xcframework/*/bin
+Apple/testbed/Python.xcframework/*/include
+Apple/testbed/Python.xcframework/*/lib
+Apple/testbed/Python.xcframework/*/Python.framework
+Apple/testbed/*Testbed.xcodeproj/project.xcworkspace
+Apple/testbed/*Testbed.xcodeproj/xcuserdata
 Mac/Makefile
 Mac/PythonLauncher/Info.plist
 Mac/PythonLauncher/Makefile
@@ -135,7 +134,6 @@ Tools/unicode/data/
 /config.log
 /config.status
 /config.status.lineno
-# hendrikmuhs/ccache-action@v1
 /.ccache
 /cross-build/
 /jit_stencils*.h
@@ -171,5 +169,6 @@ Python/frozen_modules/MANIFEST
 /python
 !/Python/
 
-# main branch only: ABI files are not checked/maintained.
-Doc/data/python*.abi
+# People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs.
+/.claude/
+CLAUDE.local.md

.pre-commit-config.yaml

@@ -1,23 +1,43 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.8
+    rev: v0.13.2
     hooks:
-      - id: ruff
+      - id: ruff-check
+        name: Run Ruff (lint) on Apple/
+        args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml]
+        files: ^Apple/
+      - id: ruff-check
         name: Run Ruff (lint) on Doc/
         args: [--exit-non-zero-on-fix]
         files: ^Doc/
-      - id: ruff
+      - id: ruff-check
         name: Run Ruff (lint) on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
-      - id: ruff
+      - id: ruff-check
         name: Run Ruff (lint) on Tools/build/
         args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
         files: ^Tools/build/
-      - id: ruff
+      - id: ruff-check
+        name: Run Ruff (lint) on Tools/i18n/
+        args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml]
+        files: ^Tools/i18n/
+      - id: ruff-check
         name: Run Ruff (lint) on Argument Clinic
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
         files: ^Tools/clinic/|Lib/test/test_clinic.py
+      - id: ruff-check
+        name: Run Ruff (lint) on Tools/peg_generator/
+        args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml]
+        files: ^Tools/peg_generator/
+      - id: ruff-check
+        name: Run Ruff (lint) on Tools/wasm/
+        args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml]
+        files: ^Tools/wasm/
+      - id: ruff-format
+        name: Run Ruff (format) on Apple/
+        args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml]
+        files: ^Apple
       - id: ruff-format
         name: Run Ruff (format) on Doc/
         args: [--check]
@@ -26,16 +46,26 @@ repos:
         name: Run Ruff (format) on Tools/build/check_warnings.py
         args: [--check, --config=Tools/build/.ruff.toml]
         files: ^Tools/build/check_warnings.py
+      - id: ruff-format
+        name: Run Ruff (format) on Tools/wasm/
+        args: [--check, --config=Tools/wasm/.ruff.toml]
+        files: ^Tools/wasm/
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.1.0
+    rev: 25.9.0
     hooks:
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/
 
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.5
+    hooks:
+      - id: remove-tabs
+        types: [python]
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -43,13 +73,17 @@ repos:
         exclude: ^Lib/test/test_tomllib/
       - id: check-yaml
       - id: end-of-file-fixer
-        types: [python]
+        types_or: [python, yaml]
         exclude: Lib/test/tokenizedata/coding20731.py
+      - id: end-of-file-fixer
+        files: '^\.github/CODEOWNERS$'
       - id: trailing-whitespace
-        types_or: [c, inc, python, rst]
+        types_or: [c, inc, python, rst, yaml]
+      - id: trailing-whitespace
+        files: '^\.github/CODEOWNERS|\.(gram)$'
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.33.0
+    rev: 0.34.0
     hooks:
       - id: check-dependabot
       - id: check-github-workflows
@@ -61,7 +95,7 @@ repos:
       - id: actionlint
 
   - repo: https://github.com/woodruffw/zizmor-pre-commit
-    rev: v1.6.0
+    rev: v1.14.1
     hooks:
       - id: zizmor
 

.readthedocs.yml

@@ -32,4 +32,3 @@ build:
     - make -C Doc venv html
     - mkdir _readthedocs
     - mv Doc/build/html _readthedocs/html
-

Android/README.md

@@ -96,10 +96,12 @@ ## Packaging
 
 ## Testing
 
-The Python test suite can be run on Linux, macOS, or Windows:
+The Python test suite can be run on Linux, macOS, or Windows.
 
-* On Linux, the emulator needs access to the KVM virtualization interface, and
-  a DISPLAY environment variable pointing at an X server. Xvfb is acceptable.
+On Linux, the emulator needs access to the KVM virtualization interface. This may
+require adding your user to a group, or changing your udev rules. On GitHub
+Actions, the test script will do this automatically using the commands shown
+[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/).
 
 The test suite can usually be run on a device with 2 GB of RAM, but this is
 borderline, so you may need to increase it to 4 GB. As of Android
@@ -156,6 +158,10 @@ ## Testing
 and architecture-specific files such as sysconfigdata, will not take effect
 until you re-run `android.py make-host` or `build`.
 
+The testbed app can also be used to test third-party packages. For more details,
+run `android.py test --help`, paying attention to the options `--site-packages`,
+`--cwd`, `-c` and `-m`.
+
 
 ## Using in your own app
 

Android/android-env.sh

@@ -3,7 +3,7 @@
 : "${HOST:?}"  # GNU target triplet
 
 # You may also override the following:
-: "${api_level:=24}"  # Minimum Android API level the build will run on
+: "${ANDROID_API_LEVEL:=24}"  # Minimum Android API level the build will run on
 : "${PREFIX:-}"  # Path in which to find required libraries
 
 
@@ -24,7 +24,7 @@ fail() {
 # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md
 #   where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.:
 #   https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md
-ndk_version=27.1.12297006
+ndk_version=27.3.13750724
 
 ndk=$ANDROID_HOME/ndk/$ndk_version
 if ! [ -e "$ndk" ]; then
@@ -43,7 +43,7 @@ fi
 toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*)
 export AR="$toolchain/bin/llvm-ar"
 export AS="$toolchain/bin/llvm-as"
-export CC="$toolchain/bin/${clang_triplet}${api_level}-clang"
+export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang"
 export CXX="${CC}++"
 export LD="$toolchain/bin/ld"
 export NM="$toolchain/bin/llvm-nm"

Android/android.py

@@ -2,7 +2,9 @@
 
 import asyncio
 import argparse
+import json
 import os
+import platform
 import re
 import shlex
 import shutil
@@ -14,7 +16,7 @@
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from glob import glob
-from os.path import basename, relpath
+from os.path import abspath, basename, relpath
 from pathlib import Path
 from subprocess import CalledProcessError
 from tempfile import TemporaryDirectory
@@ -22,9 +24,13 @@
 
 SCRIPT_NAME = Path(__file__).name
 ANDROID_DIR = Path(__file__).resolve().parent
-CHECKOUT = ANDROID_DIR.parent
+PYTHON_DIR = ANDROID_DIR.parent
+in_source_tree = (
+    ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists()
+)
+
 TESTBED_DIR = ANDROID_DIR / "testbed"
-CROSS_BUILD_DIR = CHECKOUT / "cross-build"
+CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
 
 HOSTS = ["aarch64-linux-android", "x86_64-linux-android"]
 APP_ID = "org.python.testbed"
@@ -46,7 +52,19 @@
     + (".bat" if os.name == "nt" else "")
 )
 
-logcat_started = False
+# Whether we've seen any output from Python yet.
+python_started = False
+
+# Buffer for verbose output which will be displayed only if a test fails and
+# there has been no output from Python.
+hidden_output = []
+
+
+def log_verbose(context, line, stream=sys.stdout):
+    if context.verbose:
+        stream.write(line)
+    else:
+        hidden_output.append((stream, line))
 
 
 def delete_glob(pattern):
@@ -76,39 +94,68 @@ def run(command, *, host=None, env=None, log=True, **kwargs):
     kwargs.setdefault("check", True)
     if env is None:
         env = os.environ.copy()
-    original_env = env.copy()
 
     if host:
-        env_script = ANDROID_DIR / "android-env.sh"
-        env_output = subprocess.run(
-            f"set -eu; "
-            f"HOST={host}; "
-            f"PREFIX={subdir(host)}/prefix; "
-            f". {env_script}; "
-            f"export",
-            check=True, shell=True, text=True, stdout=subprocess.PIPE
-        ).stdout
-
-        for line in env_output.splitlines():
-            # We don't require every line to match, as there may be some other
-            # output from installing the NDK.
-            if match := re.search(
-                "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
-            ):
-                key, value = match[2], match[3]
-                if env.get(key) != value:
-                    print(line)
-                    env[key] = value
-
-        if env == original_env:
-            raise ValueError(f"Found no variables in {env_script.name} output:\n"
-                             + env_output)
+        host_env = android_env(host)
+        print_env(host_env)
+        env.update(host_env)
 
     if log:
-        print(">", " ".join(map(str, command)))
+        print(">", join_command(command))
     return subprocess.run(command, env=env, **kwargs)
 
 
+# Format a command so it can be copied into a shell. Like shlex.join, but also
+# accepts arguments which are Paths, or a single string/Path outside of a list.
+def join_command(args):
+    if isinstance(args, (str, Path)):
+        return str(args)
+    else:
+        return shlex.join(map(str, args))
+
+
+# Format the environment so it can be pasted into a shell.
+def print_env(env):
+    for key, value in sorted(env.items()):
+        print(f"export {key}={shlex.quote(value)}")
+
+
+def android_env(host):
+    if host:
+        prefix = subdir(host) / "prefix"
+    else:
+        prefix = ANDROID_DIR / "prefix"
+        sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py")
+        sysconfig_filename = next(sysconfig_files).name
+        host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1]
+
+    env_script = ANDROID_DIR / "android-env.sh"
+    env_output = subprocess.run(
+        f"set -eu; "
+        f"HOST={host}; "
+        f"PREFIX={prefix}; "
+        f". {env_script}; "
+        f"export",
+        check=True, shell=True, capture_output=True, encoding='utf-8',
+    ).stdout
+
+    env = {}
+    for line in env_output.splitlines():
+        # We don't require every line to match, as there may be some other
+        # output from installing the NDK.
+        if match := re.search(
+            "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
+        ):
+            key, value = match[2], match[3]
+            if os.environ.get(key) != value:
+                env[key] = value
+
+    if not env:
+        raise ValueError(f"Found no variables in {env_script.name} output:\n"
+                         + env_output)
+    return env
+
+
 def build_python_path():
     """The path to the build Python binary."""
     build_dir = subdir("build")
@@ -127,7 +174,7 @@ def configure_build_python(context):
         clean("build")
     os.chdir(subdir("build", create=True))
 
-    command = [relpath(CHECKOUT / "configure")]
+    command = [relpath(PYTHON_DIR / "configure")]
     if context.args:
         command.extend(context.args)
     run(command)
@@ -138,13 +185,20 @@ def make_build_python(context):
     run(["make", "-j", str(os.cpu_count())])
 
 
+# To create new builds of these dependencies, usually all that's necessary is to
+# push a tag to the cpython-android-source-deps repository, and GitHub Actions
+# will do the rest.
+#
+# If you're a member of the Python core team, and you'd like to be able to push
+# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee.
 def unpack_deps(host, prefix_dir):
+    os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
-    for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4",
-                     "sqlite-3.49.1-0", "xz-5.4.6-1"]:
+    for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0",
+                     "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
-        shutil.unpack_archive(filename, prefix_dir)
+        shutil.unpack_archive(filename)
         os.remove(filename)
 
 
@@ -167,7 +221,7 @@ def configure_host_python(context):
     os.chdir(host_dir)
     command = [
         # Basic cross-compiling configuration
-        relpath(CHECKOUT / "configure"),
+        relpath(PYTHON_DIR / "configure"),
         f"--host={context.host}",
         f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}",
         f"--with-build-python={build_python_path()}",
@@ -196,9 +250,18 @@ def make_host_python(context):
     for pattern in ("include/python*", "lib/libpython*", "lib/python*"):
         delete_glob(f"{prefix_dir}/{pattern}")
 
+    # The Android environment variables were already captured in the Makefile by
+    # `configure`, and passing them again when running `make` may cause some
+    # flags to be duplicated. So we don't use the `host` argument here.
     os.chdir(host_dir)
-    run(["make", "-j", str(os.cpu_count())], host=context.host)
-    run(["make", "install", f"prefix={prefix_dir}"], host=context.host)
+    run(["make", "-j", str(os.cpu_count())])
+
+    # The `make install` output is very verbose and rarely useful, so
+    # suppress it by default.
+    run(
+        ["make", "install", f"prefix={prefix_dir}"],
+        capture_output=not context.verbose,
+    )
 
 
 def build_all(context):
@@ -217,6 +280,18 @@ def clean_all(context):
         clean(host)
 
 
+def setup_ci():
+    # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/
+    if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux":
+        run(
+            ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"],
+            input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n',
+            text=True,
+        )
+        run(["sudo", "udevadm", "control", "--reload-rules"])
+        run(["sudo", "udevadm", "trigger", "--name-match=kvm"])
+
+
 def setup_sdk():
     sdkmanager = android_home / (
         "cmdline-tools/latest/bin/sdkmanager"
@@ -228,7 +303,12 @@ def setup_sdk():
     if not all((android_home / "licenses" / path).exists() for path in [
         "android-sdk-arm-dbt-license", "android-sdk-license"
     ]):
-        run([sdkmanager, "--licenses"], text=True, input="y\n" * 100)
+        run(
+            [sdkmanager, "--licenses"],
+            text=True,
+            capture_output=True,
+            input="y\n" * 100,
+        )
 
     # Gradle may install this automatically, but we can't rely on that because
     # we need to run adb within the logcat task.
@@ -411,17 +491,19 @@ async def logcat_task(context, initial_devices):
 
     # `--pid` requires API level 24 or higher.
     args = [adb, "-s", serial, "logcat", "--pid", pid,  "--format", "tag"]
-    hidden_output = []
+    logcat_started = False
     async with async_process(
         *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
     ) as process:
         while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
             if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL):
+                logcat_started = True
                 level, message = match.groups()
             else:
-                # If the regex doesn't match, this is probably the second or
-                # subsequent line of a multi-line message. Python won't produce
-                # such messages, but other components might.
+                # If the regex doesn't match, this is either a logcat startup
+                # error, or the second or subsequent line of a multi-line
+                # message. Python won't produce multi-line messages, but other
+                # components might.
                 level, message = None, line
 
             # Exclude high-volume messages which are rarely useful.
@@ -441,25 +523,22 @@ async def logcat_task(context, initial_devices):
             # tag indicators from Python's stdout and stderr.
             for prefix in ["python.stdout: ", "python.stderr: "]:
                 if message.startswith(prefix):
-                    global logcat_started
-                    logcat_started = True
+                    global python_started
+                    python_started = True
                     stream.write(message.removeprefix(prefix))
                     break
             else:
-                if context.verbose:
-                    # Non-Python messages add a lot of noise, but they may
-                    # sometimes help explain a failure.
-                    stream.write(line)
-                else:
-                    hidden_output.append(line)
+                # Non-Python messages add a lot of noise, but they may
+                # sometimes help explain a failure.
+                log_verbose(context, line, stream)
 
         # If the device disconnects while logcat is running, which always
         # happens in --managed mode, some versions of adb return non-zero.
         # Distinguish this from a logcat startup error by checking whether we've
-        # received a message from Python yet.
+        # received any logcat messages yet.
         status = await wait_for(process.wait(), timeout=1)
         if status != 0 and not logcat_started:
-            raise CalledProcessError(status, args, "".join(hidden_output))
+            raise CalledProcessError(status, args)
 
 
 def stop_app(serial):
@@ -474,12 +553,38 @@ async def gradle_task(context):
         task_prefix = "connected"
         env["ANDROID_SERIAL"] = context.connected
 
+    if context.ci_mode:
+        context.args[0:0] = [
+            # See _add_ci_python_opts in libregrtest/main.py.
+            "-W", "error", "-bb", "-E",
+
+            # Randomization is disabled because order-dependent failures are
+            # much less likely to pass on a rerun in single-process mode.
+            "-m", "test",
+            f"--{context.ci_mode}-ci", "--single-process", "--no-randomize"
+        ]
+
+    if not any(arg in context.args for arg in ["-c", "-m"]):
+        context.args[0:0] = ["-m", "test"]
+
     args = [
         gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest",
-        "-Pandroid.testInstrumentationRunnerArguments.pythonArgs="
-        + shlex.join(context.args),
+    ] + [
+        f"-P{name}={value}"
+        for name, value in [
+            ("python.sitePackages", context.site_packages),
+            ("python.cwd", context.cwd),
+            (
+                "android.testInstrumentationRunnerArguments.pythonArgs",
+                json.dumps(context.args),
+            ),
+        ]
+        if value
     ]
-    hidden_output = []
+    if context.verbose >= 2:
+        args.append("--info")
+    log_verbose(context, f"> {join_command(args)}\n")
+
     try:
         async with async_process(
             *args, cwd=TESTBED_DIR, env=env,
@@ -488,10 +593,10 @@ async def gradle_task(context):
             while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
                 # Gradle may take several minutes to install SDK packages, so
                 # it's worth showing those messages even in non-verbose mode.
-                if context.verbose or line.startswith('Preparing "Install'):
+                if line.startswith('Preparing "Install'):
                     sys.stdout.write(line)
                 else:
-                    hidden_output.append(line)
+                    log_verbose(context, line)
 
             status = await wait_for(process.wait(), timeout=1)
             if status == 0:
@@ -499,17 +604,13 @@ async def gradle_task(context):
             else:
                 raise CalledProcessError(status, args)
     finally:
-        # If logcat never started, then something has gone badly wrong, so the
-        # user probably wants to see the Gradle output even in non-verbose mode.
-        if hidden_output and not logcat_started:
-            sys.stdout.write("".join(hidden_output))
-
         # Gradle does not stop the tests when interrupted.
         if context.connected:
             stop_app(context.connected)
 
 
 async def run_testbed(context):
+    setup_ci()
     setup_sdk()
     setup_testbed()
 
@@ -533,6 +634,12 @@ async def run_testbed(context):
     except* MySystemExit as e:
         raise SystemExit(*e.exceptions[0].args) from None
     except* CalledProcessError as e:
+        # If Python produced no output, then the user probably wants to see the
+        # verbose output to explain why the test failed.
+        if not python_started:
+            for stream, line in hidden_output:
+                stream.write(line)
+
         # Extract it from the ExceptionGroup so it can be handled by `main`.
         raise e.exceptions[0]
 
@@ -597,11 +704,64 @@ def package(context):
                     else:
                         shutil.copy2(src, dst, follow_symlinks=False)
 
+        # Strip debug information.
+        if not context.debug:
+            so_files = glob(f"{temp_dir}/**/*.so", recursive=True)
+            run([android_env(context.host)["STRIP"], *so_files], log=False)
+
         dist_dir = subdir(context.host, "dist", create=True)
         package_path = shutil.make_archive(
             f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir
         )
         print(f"Wrote {package_path}")
+        return package_path
+
+
+def ci(context):
+    for step in [
+        configure_build_python,
+        make_build_python,
+        configure_host_python,
+        make_host_python,
+        package,
+    ]:
+        caption = (
+            step.__name__.replace("_", " ")
+            .capitalize()
+            .replace("python", "Python")
+        )
+        print(f"::group::{caption}")
+        result = step(context)
+        if step is package:
+            package_path = result
+        print("::endgroup::")
+
+    if (
+        "GITHUB_ACTIONS" in os.environ
+        and (platform.system(), platform.machine()) != ("Linux", "x86_64")
+    ):
+        print(
+            "Skipping tests: GitHub Actions does not support the Android "
+            "emulator on this platform."
+        )
+    else:
+        with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
+            print("::group::Tests")
+
+            # Prove the package is self-contained by using it to run the tests.
+            shutil.unpack_archive(package_path, temp_dir)
+            launcher_args = [
+                "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci"
+            ]
+            run(
+                ["./android.py", "test", *launcher_args],
+                cwd=temp_dir
+            )
+            print("::endgroup::")
+
+
+def env(context):
+    print_env(android_env(getattr(context, "host", None)))
 
 
 # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated
@@ -615,45 +775,54 @@ def signal_handler(*args):
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    subcommands = parser.add_subparsers(dest="subcommand")
+    subcommands = parser.add_subparsers(dest="subcommand", required=True)
+
+    def add_parser(*args, **kwargs):
+        parser = subcommands.add_parser(*args, **kwargs)
+        parser.add_argument(
+            "-v", "--verbose", action="count", default=0,
+            help="Show verbose output. Use twice to be even more verbose.")
+        return parser
 
     # Subcommands
-    build = subcommands.add_parser("build", help="Build everything")
-    configure_build = subcommands.add_parser("configure-build",
-                                             help="Run `configure` for the "
-                                             "build Python")
-    make_build = subcommands.add_parser("make-build",
-                                        help="Run `make` for the build Python")
-    configure_host = subcommands.add_parser("configure-host",
-                                            help="Run `configure` for Android")
-    make_host = subcommands.add_parser("make-host",
-                                       help="Run `make` for Android")
-    subcommands.add_parser(
-        "clean", help="Delete all build and prefix directories")
-    subcommands.add_parser(
-        "build-testbed", help="Build the testbed app")
-    test = subcommands.add_parser(
-        "test", help="Run the test suite")
-    package = subcommands.add_parser("package", help="Make a release package")
+    build = add_parser(
+        "build", help="Run configure-build, make-build, configure-host and "
+        "make-host")
+    configure_build = add_parser(
+        "configure-build", help="Run `configure` for the build Python")
+    add_parser(
+        "make-build", help="Run `make` for the build Python")
+    configure_host = add_parser(
+        "configure-host", help="Run `configure` for Android")
+    make_host = add_parser(
+        "make-host", help="Run `make` for Android")
+
+    add_parser("clean", help="Delete all build directories")
+    add_parser("build-testbed", help="Build the testbed app")
+    test = add_parser("test", help="Run the testbed app")
+    package = add_parser("package", help="Make a release package")
+    ci = add_parser("ci", help="Run build, package and test")
+    env = add_parser("env", help="Print environment variables")
 
     # Common arguments
-    for subcommand in build, configure_build, configure_host:
+    for subcommand in [build, configure_build, configure_host, ci]:
         subcommand.add_argument(
             "--clean", action="store_true", default=False, dest="clean",
-            help="Delete the relevant build and prefix directories first")
-    for subcommand in [build, configure_host, make_host, package]:
+            help="Delete the relevant build directories first")
+
+    host_commands = [build, configure_host, make_host, package, ci]
+    if in_source_tree:
+        host_commands.append(env)
+    for subcommand in host_commands:
         subcommand.add_argument(
             "host", metavar="HOST", choices=HOSTS,
             help="Host triplet: choices=[%(choices)s]")
-    for subcommand in build, configure_build, configure_host:
+
+    for subcommand in [build, configure_build, configure_host, ci]:
         subcommand.add_argument("args", nargs="*",
                                 help="Extra arguments to pass to `configure`")
 
     # Test arguments
-    test.add_argument(
-        "-v", "--verbose", action="count", default=0,
-        help="Show Gradle output, and non-Python logcat messages. "
-        "Use twice to include high-volume messages which are rarely useful.")
     device_group = test.add_mutually_exclusive_group(required=True)
     device_group.add_argument(
         "--connected", metavar="SERIAL", help="Run on a connected device. "
@@ -661,9 +830,34 @@ def parse_args():
     device_group.add_argument(
         "--managed", metavar="NAME", help="Run on a Gradle-managed device. "
         "These are defined in `managedDevices` in testbed/app/build.gradle.kts.")
+
     test.add_argument(
-        "args", nargs="*", help=f"Arguments for `python -m test`. "
-        f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.")
+        "--site-packages", metavar="DIR", type=abspath,
+        help="Directory to copy as the app's site-packages.")
+    test.add_argument(
+        "--cwd", metavar="DIR", type=abspath,
+        help="Directory to copy as the app's working directory.")
+    test.add_argument(
+        "args", nargs="*", help=f"Python command-line arguments. "
+        f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. "
+        f"If neither -c nor -m are included, `-m test` will be prepended, "
+        f"which will run Python's own test suite.")
+
+    # Package arguments.
+    for subcommand in [package, ci]:
+        subcommand.add_argument(
+            "-g", action="store_true", default=False, dest="debug",
+            help="Include debug information in package")
+
+    # CI arguments
+    for subcommand in [test, ci]:
+        group = subcommand.add_mutually_exclusive_group(required=subcommand is ci)
+        group.add_argument(
+            "--fast-ci", action="store_const", dest="ci_mode", const="fast",
+            help="Add test arguments for GitHub Actions")
+        group.add_argument(
+            "--slow-ci", action="store_const", dest="ci_mode", const="slow",
+            help="Add test arguments for buildbots")
 
     return parser.parse_args()
 
@@ -688,6 +882,8 @@ def main():
         "build-testbed": build_testbed,
         "test": run_testbed,
         "package": package,
+        "ci": ci,
+        "env": env,
     }
 
     try:
@@ -702,20 +898,17 @@ def main():
 def print_called_process_error(e):
     for stream_name in ["stdout", "stderr"]:
         content = getattr(e, stream_name)
+        if isinstance(content, bytes):
+            content = content.decode(*DECODE_ARGS)
         stream = getattr(sys, stream_name)
         if content:
             stream.write(content)
             if not content.endswith("\n"):
                 stream.write("\n")
 
-    # Format the command so it can be copied into a shell. shlex uses single
-    # quotes, so we surround the whole command with double quotes.
-    args_joined = (
-        e.cmd if isinstance(e.cmd, str)
-        else " ".join(shlex.quote(str(arg)) for arg in e.cmd)
-    )
+    # shlex uses single quotes, so we surround the command with double quotes.
     print(
-        f'Command "{args_joined}" returned exit status {e.returncode}'
+        f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}'
     )
 
 

Android/testbed/app/build.gradle.kts

@@ -47,7 +47,7 @@ for ((i, prefix) in prefixes.withIndex()) {
     val libDir = file("$prefix/lib")
     val version = run {
         for (filename in libDir.list()!!) {
-            """python(\d+\.\d+)""".toRegex().matchEntire(filename)?.let {
+            """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let {
                 return@run it.groupValues[1]
             }
         }
@@ -64,9 +64,10 @@ for ((i, prefix) in prefixes.withIndex()) {
     val libPythonDir = file("$libDir/python$pythonVersion")
     val triplet = run {
         for (filename in libPythonDir.list()!!) {
-            """_sysconfigdata__android_(.+).py""".toRegex().matchEntire(filename)?.let {
-                return@run it.groupValues[1]
-            }
+            """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex()
+                .matchEntire(filename)?.let {
+                    return@run it.groupValues[1]
+                }
         }
         throw GradleException("Failed to find Python triplet in $libPythonDir")
     }
@@ -85,7 +86,7 @@ android {
 
         minSdk = androidEnvFile.useLines {
             for (line in it) {
-                """api_level:=(\d+)""".toRegex().find(line)?.let {
+                """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let {
                     return@useLines it.groupValues[1].toInt()
                 }
             }
@@ -205,11 +206,29 @@ androidComponents.onVariants { variant ->
 
                 into("site-packages") {
                     from("$projectDir/src/main/python")
+
+                    val sitePackages = findProperty("python.sitePackages") as String?
+                    if (!sitePackages.isNullOrEmpty()) {
+                        if (!file(sitePackages).exists()) {
+                            throw GradleException("$sitePackages does not exist")
+                        }
+                        from(sitePackages)
+                    }
                 }
 
                 duplicatesStrategy = DuplicatesStrategy.EXCLUDE
                 exclude("**/__pycache__")
             }
+
+            into("cwd") {
+                val cwd = findProperty("python.cwd") as String?
+                if (!cwd.isNullOrEmpty()) {
+                    if (!file(cwd).exists()) {
+                        throw GradleException("$cwd does not exist")
+                    }
+                    from(cwd)
+                }
+            }
         }
     }
 

Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt

@@ -17,11 +17,11 @@ class PythonSuite {
     fun testPython() {
         val start = System.currentTimeMillis()
         try {
-            val context =
+            val status = PythonTestRunner(
                 InstrumentationRegistry.getInstrumentation().targetContext
-            val args =
-                InstrumentationRegistry.getArguments().getString("pythonArgs", "")
-            val status = PythonTestRunner(context).run(args)
+            ).run(
+                InstrumentationRegistry.getArguments().getString("pythonArgs")!!,
+            )
             assertEquals(0, status)
         } finally {
             // Make sure the process lives long enough for the test script to

Android/testbed/app/src/main/c/main_activity.c

@@ -3,6 +3,7 @@
 #include <jni.h>
 #include <pthread.h>
 #include <Python.h>
+#include <signal.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
@@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) {
         message);
 }
 
+static void throw_errno(JNIEnv *env, const char *error_prefix) {
+    char error_message[1024];
+    snprintf(error_message, sizeof(error_message),
+             "%s: %s", error_prefix, strerror(errno));
+    throw_runtime_exception(env, error_message);
+}
+
 
 // --- Stdio redirection ------------------------------------------------------
 
@@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL
     for (StreamInfo *si = STREAMS; si->file; si++) {
         char *error_prefix;
         if ((error_prefix = redirect_stream(si))) {
-            char error_message[1024];
-            snprintf(error_message, sizeof(error_message),
-                     "%s: %s", error_prefix, strerror(errno));
-            throw_runtime_exception(env, error_message);
+            throw_errno(env, error_prefix);
             return;
         }
     }
@@ -107,13 +112,38 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL
 
 // --- Python initialization ---------------------------------------------------
 
-static PyStatus set_config_string(
-    JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value
-) {
-    const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL);
-    PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8);
-    (*env)->ReleaseStringUTFChars(env, value, value_utf8);
-    return status;
+static char *init_signals() {
+    // Some tests use SIGUSR1, but that's blocked by default in an Android app in
+    // order to make it available to `sigwait` in the Signal Catcher thread.
+    // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc).
+    // That thread's functionality is only useful for debugging the JVM, so disabling
+    // it should not weaken the tests.
+    //
+    // There's no safe way of stopping the thread completely (#123982), but simply
+    // unblocking SIGUSR1 is enough to fix most tests.
+    //
+    // However, in tests that generate multiple different signals in quick
+    // succession, it's possible for SIGUSR1 to arrive while the main thread is busy
+    // running the C-level handler for a different signal. In that case, the SIGUSR1
+    // may be sent to the Signal Catcher thread instead, which will generate a log
+    // message containing the text "reacting to signal".
+    //
+    // Such tests may need to be changed in one of the following ways:
+    //   * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in
+    //     test_signal.py).
+    //   * Send the signal to a specific thread rather than the whole process (e.g.
+    //     test_signals in test_threadsignals.py.
+    sigset_t set;
+    if (sigemptyset(&set)) {
+        return "sigemptyset";
+    }
+    if (sigaddset(&set, SIGUSR1)) {
+        return "sigaddset";
+    }
+    if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) {
+        return "pthread_sigmask";
+    }
+    return NULL;
 }
 
 static void throw_status(JNIEnv *env, PyStatus status) {
@@ -121,27 +151,47 @@ static void throw_status(JNIEnv *env, PyStatus status) {
 }
 
 JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython(
-    JNIEnv *env, jobject obj, jstring home, jstring runModule
+    JNIEnv *env, jobject obj, jstring home, jarray args
 ) {
+    const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL);
+    char cwd[PATH_MAX];
+    snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd");
+    if (chdir(cwd)) {
+        throw_errno(env, "chdir");
+        return 1;
+    }
+
+    char *error_prefix;
+    if ((error_prefix = init_signals())) {
+        throw_errno(env, error_prefix);
+        return 1;
+    }
+
     PyConfig config;
     PyStatus status;
-    PyConfig_InitIsolatedConfig(&config);
+    PyConfig_InitPythonConfig(&config);
 
-    status = set_config_string(env, &config, &config.home, home);
-    if (PyStatus_Exception(status)) {
+    jsize argc = (*env)->GetArrayLength(env, args);
+    const char *argv[argc + 1];
+    for (int i = 0; i < argc; i++) {
+        jobject arg = (*env)->GetObjectArrayElement(env, args, i);
+        argv[i] = (*env)->GetStringUTFChars(env, arg, NULL);
+    }
+    argv[argc] = NULL;
+
+    // PyConfig_SetBytesArgv "must be called before other methods, since the
+    // preinitialization configuration depends on command line arguments"
+    if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) {
         throw_status(env, status);
         return 1;
     }
 
-    status = set_config_string(env, &config, &config.run_module, runModule);
+    status = PyConfig_SetBytesString(&config, &config.home, home_utf8);
     if (PyStatus_Exception(status)) {
         throw_status(env, status);
         return 1;
     }
 
-    // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored.
-    config.install_signal_handlers = 1;
-
     status = Py_InitializeFromConfig(&config);
     if (PyStatus_Exception(status)) {
         throw_status(env, status);

Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt

@@ -5,6 +5,7 @@ import android.os.*
 import android.system.Os
 import android.widget.TextView
 import androidx.appcompat.app.*
+import org.json.JSONArray
 import java.io.*
 
 
@@ -15,18 +16,25 @@ class MainActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(R.layout.activity_main)
-        val status = PythonTestRunner(this).run("-W -uall")
+        val status = PythonTestRunner(this).run("""["-m", "test", "-W", "-uall"]""")
         findViewById<TextView>(R.id.tvHello).text = "Exit status $status"
     }
 }
 
 
 class PythonTestRunner(val context: Context) {
-    /** @param args Extra arguments for `python -m test`.
-     * @return The Python exit status: zero if the tests passed, nonzero if
-     * they failed. */
-    fun run(args: String = "") : Int {
-        Os.setenv("PYTHON_ARGS", args, true)
+    /** Run Python.
+     *
+     * @param args Python command-line, encoded as JSON.
+     * @return The Python exit status: zero on success, nonzero on failure. */
+    fun run(args: String) : Int {
+        // We leave argument 0 as an empty string, which is a placeholder for the
+        // executable name in embedded mode.
+        val argsJsonArray = JSONArray(args)
+        val argsStringArray = Array<String>(argsJsonArray.length() + 1) { it -> ""}
+        for (i in 0..<argsJsonArray.length()) {
+            argsStringArray[i + 1] = argsJsonArray.getString(i)
+        }
 
         // Python needs this variable to help it find the temporary directory,
         // but Android only sets it on API level 33 and later.
@@ -35,9 +43,7 @@ class PythonTestRunner(val context: Context) {
         val pythonHome = extractAssets()
         System.loadLibrary("main_activity")
         redirectStdioToLogcat()
-
-        // The main module is in src/main/python/main.py.
-        return runPython(pythonHome.toString(), "main")
+        return runPython(pythonHome.toString(), argsStringArray)
     }
 
     private fun extractAssets() : File {
@@ -46,6 +52,13 @@ class PythonTestRunner(val context: Context) {
             throw RuntimeException("Failed to delete $pythonHome")
         }
         extractAssetDir("python", context.filesDir)
+
+        // Empty directories are lost in the asset packing/unpacking process.
+        val cwd = File(pythonHome, "cwd")
+        if (!cwd.exists()) {
+            cwd.mkdir()
+        }
+
         return pythonHome
     }
 
@@ -75,5 +88,5 @@ class PythonTestRunner(val context: Context) {
     }
 
     private external fun redirectStdioToLogcat()
-    private external fun runPython(home: String, runModule: String) : Int
+    private external fun runPython(home: String, args: Array<String>) : Int
 }

Android/testbed/app/src/main/python/main.py

@@ -1,32 +0,0 @@
-import os
-import runpy
-import shlex
-import signal
-import sys
-
-# Some tests use SIGUSR1, but that's blocked by default in an Android app in
-# order to make it available to `sigwait` in the Signal Catcher thread.
-# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc).
-# That thread's functionality is only useful for debugging the JVM, so disabling
-# it should not weaken the tests.
-#
-# There's no safe way of stopping the thread completely (#123982), but simply
-# unblocking SIGUSR1 is enough to fix most tests.
-#
-# However, in tests that generate multiple different signals in quick
-# succession, it's possible for SIGUSR1 to arrive while the main thread is busy
-# running the C-level handler for a different signal. In that case, the SIGUSR1
-# may be sent to the Signal Catcher thread instead, which will generate a log
-# message containing the text "reacting to signal".
-#
-# Such tests may need to be changed in one of the following ways:
-#   * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in
-#     test_signal.py).
-#   * Send the signal to a specific thread rather than the whole process (e.g.
-#     test_signals in test_threadsignals.py.
-signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1])
-
-sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"])
-
-# The test module will call sys.exit to indicate whether the tests passed.
-runpy.run_module("test")

Android/testbed/build.gradle.kts

@@ -1,5 +1,5 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 plugins {
-    id("com.android.application") version "8.6.1" apply false
+    id("com.android.application") version "8.10.0" apply false
     id("org.jetbrains.kotlin.android") version "1.9.22" apply false
 }

Android/testbed/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 #Mon Feb 19 20:29:06 GMT 2024
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

Apple/.ruff.toml

@@ -0,0 +1,22 @@
+extend = "../.ruff.toml"  # Inherit the project-wide settings
+
+[format]
+preview = true
+docstring-code-format = true
+
+[lint]
+select = [
+    "C4",      # flake8-comprehensions
+    "E",       # pycodestyle
+    "F",       # pyflakes
+    "I",       # isort
+    "ISC",     # flake8-implicit-str-concat
+    "LOG",     # flake8-logging
+    "PGH",     # pygrep-hooks
+    "PT",      # flake8-pytest-style
+    "PYI",     # flake8-pyi
+    "RUF100",  # Ban unused `# noqa` comments
+    "UP",      # pyupgrade
+    "W",       # pycodestyle
+    "YTT",     # flake8-2020
+]

Apple/iOS/README.md

@@ -0,0 +1,339 @@
+# Python on iOS README
+
+**iOS support is [tier 3](https://peps.python.org/pep-0011/#tier-3).**
+
+This document provides a quick overview of some iOS specific features in the
+Python distribution.
+
+These instructions are only needed if you're planning to compile Python for iOS
+yourself. Most users should *not* need to do this. If you're looking to
+experiment with writing an iOS app in Python, tools such as [BeeWare's
+Briefcase](https://briefcase.readthedocs.io) and [Kivy's
+Buildozer](https://buildozer.readthedocs.io) will provide a much more
+approachable user experience.
+
+## Compilers for building on iOS
+
+Building for iOS requires the use of Apple's Xcode tooling. It is strongly
+recommended that you use the most recent stable release of Xcode. This will
+require the use of the most (or second-most) recently released macOS version,
+as Apple does not maintain Xcode for older macOS versions. The Xcode Command
+Line Tools are not sufficient for iOS development; you need a *full* Xcode
+install.
+
+If you want to run your code on the iOS simulator, you'll also need to install
+an iOS Simulator Platform. You should be prompted to select an iOS Simulator
+Platform when you first run Xcode. Alternatively, you can add an iOS Simulator
+Platform by selecting an open the Platforms tab of the Xcode Settings panel.
+
+## Building Python on iOS
+
+### ABIs and Architectures
+
+iOS apps can be deployed on physical devices, and on the iOS simulator. Although
+the API used on these devices is identical, the ABI is different - you need to
+link against different libraries for an iOS device build (`iphoneos`) or an
+iOS simulator build (`iphonesimulator`).
+
+Apple uses the `XCframework` format to allow specifying a single dependency
+that supports multiple ABIs. An `XCframework` is a wrapper around multiple
+ABI-specific frameworks that share a common API.
+
+iOS can also support different CPU architectures within each ABI. At present,
+there is only a single supported architecture on physical devices - ARM64.
+However, the *simulator* supports 2 architectures - ARM64 (for running on Apple
+Silicon machines), and x86_64 (for running on older Intel-based machines).
+
+To support multiple CPU architectures on a single platform, Apple uses a "fat
+binary" format - a single physical file that contains support for multiple
+architectures. It is possible to compile and use a "thin" single architecture
+version of a binary for testing purposes; however, the "thin" binary will not be
+portable to machines using other architectures.
+
+### Building a multi-architecture iOS XCframework
+
+The `Apple` subfolder of the Python repository acts as a build script that
+can be used to coordinate the compilation of a complete iOS XCframework. To use
+it, run::
+
+  python Apple build iOS
+
+This will:
+
+* Configure and compile a version of Python to run on the build machine
+* Download pre-compiled binary dependencies for each platform
+* Configure and build a `Python.framework` for each required architecture and
+  iOS SDK
+* Merge the multiple `Python.framework` folders into a single `Python.xcframework`
+* Produce a `.tar.gz` archive in the `cross-build/dist` folder containing
+  the `Python.xcframework`, plus a copy of the Testbed app pre-configured to
+  use the XCframework.
+
+The `Apple` build script has other entry points that will perform the
+individual parts of the overall `build` target, plus targets to test the
+build, clean the `cross-build` folder of iOS build products, and perform a
+complete "build and test" CI run. The `--clean` flag can also be used on
+individual commands to ensure that a stale build product are removed before
+building.
+
+### Building a single-architecture framework
+
+If you're using the `Apple` build script, you won't need to build
+individual frameworks. However, if you do need to manually configure an iOS
+Python build for a single framework, the following options are available.
+
+#### iOS specific arguments to configure
+
+* `--enable-framework[=DIR]`
+
+  This argument specifies the location where the Python.framework will be
+  installed. If `DIR` is not specified, the framework will be installed into
+  a subdirectory of the `iOS/Frameworks` folder.
+
+  This argument *must* be provided when configuring iOS builds. iOS does not
+  support non-framework builds.
+
+* `--with-framework-name=NAME`
+
+  Specify the name for the Python framework; defaults to `Python`.
+
+  > [!NOTE]
+  > Unless you know what you're doing, changing the name of the Python
+  > framework on iOS is not advised. If you use this option, you won't be able
+  > to run the `Apple` build script without making significant manual
+  > alterations, and you won't be able to use any binary packages unless you
+  > compile them yourself using your own framework name.
+
+#### Building Python for iOS
+
+The Python build system will create a `Python.framework` that supports a
+*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a
+framework to contain non-library content, so the iOS build will produce a
+`bin` and `lib` folder in the same output folder as `Python.framework`.
+The `lib` folder will be needed at runtime to support the Python library.
+
+If you want to use Python in a real iOS project, you need to produce multiple
+`Python.framework` builds, one for each ABI and architecture. iOS builds of
+Python *must* be constructed as framework builds. To support this, you must
+provide the `--enable-framework` flag when configuring the build. The build
+also requires the use of cross-compilation. The minimal commands for building
+Python for the ARM64 iOS simulator will look something like:
+```
+export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin"
+./configure \
+    --enable-framework \
+    --host=arm64-apple-ios-simulator \
+    --build=arm64-apple-darwin \
+    --with-build-python=/path/to/python.exe
+make
+make install
+```
+
+In this invocation:
+
+* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the
+  compilers and linkers needed by the build. Xcode requires the use of `xcrun`
+  to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the
+  result passed to `configure`, these results can embed user- and
+  version-specific paths into the sysconfig data, which limits the portability
+  of the compiled Python. Alternatively, if `xcrun` is used *as* the compiler,
+  it requires that compiler variables like `CC` include spaces, which can
+  cause significant problems with many C configuration systems which assume that
+  `CC` will be a single executable.
+
+  To work around this problem, the `Apple/iOS/Resources/bin` folder contains some
+  wrapper scripts that present as simple compilers and linkers, but wrap
+  underlying calls to `xcrun`. This allows configure to use a `CC`
+  definition without spaces, and without user- or version-specific paths, while
+  retaining the ability to adapt to the local Xcode install. These scripts are
+  included in the `bin` directory of an iOS install.
+
+  These scripts will, by default, use the currently active Xcode installation.
+  If you want to use a different Xcode installation, you can use
+  `xcode-select` to set a new default Xcode globally, or you can use the
+  `DEVELOPER_DIR` environment variable to specify an Xcode install. The
+  scripts will use the default `iphoneos`/`iphonesimulator` SDK version for
+  the select Xcode install; if you want to use a different SDK, you can set the
+  `IOS_SDK_VERSION` environment variable. (e.g, setting
+  `IOS_SDK_VERSION=17.1` would cause the scripts to use the `iphoneos17.1`
+  and `iphonesimulator17.1` SDKs, regardless of the Xcode default.)
+
+  The path has also been cleared of any user customizations. A common source of
+  bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS
+  build. Resetting the path to a known "bare bones" value is the easiest way to
+  avoid these problems.
+
+* `--host` is the architecture and ABI that you want to build, in GNU compiler
+  triple format. This will be one of:
+
+  - `arm64-apple-ios` for ARM64 iOS devices.
+  - `arm64-apple-ios-simulator` for the iOS simulator running on Apple
+    Silicon devices.
+  - `x86_64-apple-ios-simulator` for the iOS simulator running on Intel
+    devices.
+
+* `--build` is the GNU compiler triple for the machine that will be running
+  the compiler. This is one of:
+
+  - `arm64-apple-darwin` for Apple Silicon devices.
+  - `x86_64-apple-darwin` for Intel devices.
+
+* `/path/to/python.exe` is the path to a Python binary on the machine that
+  will be running the compiler. This is needed because the Python compilation
+  process involves running some Python code. On a normal desktop build of
+  Python, you can compile a python interpreter and then use that interpreter to
+  run Python code. However, the binaries produced for iOS won't run on macOS, so
+  you need to provide an external Python interpreter. This interpreter must be
+  the same version as the Python that is being compiled. To be completely safe,
+  this should be the *exact* same commit hash. However, the longer a Python
+  release has been stable, the more likely it is that this constraint can be
+  relaxed - the same micro version will often be sufficient.
+
+* The `install` target for iOS builds is slightly different to other
+  platforms. On most platforms, `make install` will install the build into
+  the final runtime location. This won't be the case for iOS, as the final
+  runtime location will be on a physical device.
+
+  However, you still need to run the `install` target for iOS builds, as it
+  performs some final framework assembly steps. The location specified with
+  `--enable-framework` will be the location where `make install` will
+  assemble the complete iOS framework. This completed framework can then
+  be copied and relocated as required.
+
+For a full CPython build, you also need to specify the paths to iOS builds of
+the binary libraries that CPython depends on (such as XZ, LibFFI and OpenSSL).
+This can be done by defining library specific environment variables (such as
+`LIBLZMA_CFLAGS`, `LIBLZMA_LIBS`), and the `--with-openssl` configure
+option. Versions of these libraries pre-compiled for iOS can be found in [this
+repository](https://github.com/beeware/cpython-apple-source-deps/releases).
+LibFFI is especially important, as many parts of the standard library
+(including the `platform`, `sysconfig` and `webbrowser` modules) require
+the use of the `ctypes` module at runtime.
+
+By default, Python will be compiled with an iOS deployment target (i.e., the
+minimum supported iOS version) of 13.0. To specify a different deployment
+target, provide the version number as part of the `--host` argument - for
+example, `--host=arm64-apple-ios15.4-simulator` would compile an ARM64
+simulator build with a deployment target of 15.4.
+
+## Testing Python on iOS
+
+### Testing a multi-architecture framework
+
+Once you have a built an XCframework, you can test that framework by running:
+
+  $ python Apple test iOS
+
+This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or
+iPhone 16e, or similar), and run the test suite on the most recent version of
+iOS that is available. You can specify a simulator using the `--simulator`
+command line argument, providing the name of the simulator (e.g., `--simulator
+'iPhone 16 Pro'`). You can also use this argument to control the OS version used
+for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the
+tests on an iPhone 16 Pro running iOS 18.2.
+
+If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS`
+environment variable will be exposed to the iOS process at runtime.
+
+### Testing a single-architecture framework
+
+The `Apple/testbed` folder that contains an Xcode project that is able to run
+the Python test suite on Apple platforms. This project converts the Python test
+suite into a single test case in Xcode's XCTest framework. The single XCTest
+passes if the test suite passes.
+
+To run the test suite, configure a Python build for an iOS simulator (i.e.,
+`--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator`
+), specifying a framework build (i.e. `--enable-framework`). Ensure that your
+`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and
+exclude any non-iOS tools, then run:
+```
+make all
+make install
+make testios
+```
+
+This will:
+
+* Build an iOS framework for your chosen architecture;
+* Finalize the single-platform framework;
+* Make a clean copy of the testbed project;
+* Install the Python iOS framework into the copy of the testbed project; and
+* Run the test suite on an "entry-level device" simulator (i.e., an iPhone SE,
+  iPhone 16e, or a similar).
+
+On success, the test suite will exit and report successful completion of the
+test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15
+minutes to run; a couple of extra minutes is required to compile the testbed
+project, and then boot and prepare the iOS simulator.
+
+### Debugging test failures
+
+Running `python Apple test iOS` generates a standalone version of the
+`Apple/testbed` project, and runs the full test suite. It does this using
+`Apple/testbed` itself - the folder is an executable module that can be used
+to create and run a clone of the testbed project. The standalone version of the
+testbed will be created in a directory named
+`cross-build/iOS-testbed.<timestamp>`.
+
+You can generate your own standalone testbed instance by running:
+```
+python cross-build/iOS/testbed clone my-testbed
+```
+
+In this invocation, `my-testbed` is the name of the folder for the new
+testbed clone.
+
+If you've built your own XCframework, or you only want to test a single architecture,
+you can construct a standalone testbed instance by running:
+```
+python Apple/testbed clone --platform iOS --framework <path/to/framework> my-testbed
+```
+
+The framework path can be the path path to a `Python.xcframework`, or the
+path to a folder that contains a single-platform `Python.framework`.
+
+You can then use the `my-testbed` folder to run the Python test suite,
+passing in any command line arguments you may require. For example, if you're
+trying to diagnose a failure in the `os` module, you might run:
+```
+python my-testbed run -- test -W test_os
+```
+
+This is the equivalent of running `python -m test -W test_os` on a desktop
+Python build. Any arguments after the `--` will be passed to testbed as if
+they were arguments to `python -m` on a desktop machine.
+
+### Testing in Xcode
+
+You can also open the testbed project in Xcode by running:
+```
+open my-testbed/iOSTestbed.xcodeproj
+```
+
+This will allow you to use the full Xcode suite of tools for debugging.
+
+The arguments used to run the test suite are defined as part of the test plan.
+To modify the test plan, select the test plan node of the project tree (it
+should be the first child of the root node), and select the "Configurations"
+tab. Modify the "Arguments Passed On Launch" value to change the testing
+arguments.
+
+The test plan also disables parallel testing, and specifies the use of the
+`Testbed.lldbinit` file for providing configuration of the debugger. The
+default debugger configuration disables automatic breakpoints on the
+`SIGINT`, `SIGUSR1`, `SIGUSR2`, and `SIGXFSZ` signals.
+
+### Testing on an iOS device
+
+To test on an iOS device, the app needs to be signed with known developer
+credentials. To obtain these credentials, you must have an iOS Developer
+account, and your Xcode install will need to be logged into your account (see
+the Accounts tab of the Preferences dialog).
+
+Once the project is open, and you're signed into your Apple Developer account,
+select the root node of the project tree (labeled "iOSTestbed"), then the
+"Signing & Capabilities" tab in the details page. Select a development team
+(this will likely be your own name), and plug in a physical device to your
+macOS machine with a USB cable. You should then be able to select your physical
+device from the list of targets in the pulldown in the Xcode titlebar.

Apple/iOS/Resources/bin/arm64-apple-ios-ar

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-clang

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-clang++

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-cpp

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@"

Apple/iOS/Resources/bin/arm64-apple-ios-strip

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@"

Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@"

Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"

Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@"

Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@"

Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip

@@ -0,0 +1,2 @@
+#!/bin/sh
+xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@"

Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist

@@ -19,7 +19,7 @@
 		<string>iPhoneOS</string>
 	</array>
 	<key>MinimumOSVersion</key>
-	<string>12.0</string>
+	<string>13.0</string>
 	<key>CFBundleVersion</key>
 	<string>1</string>
 </dict>

Apple/testbed/Python.xcframework/build/utils.sh

@@ -0,0 +1,151 @@
+# Utility methods for use in an Xcode project.
+#
+# An iOS XCframework cannot include any content other than the library binary
+# and relevant metadata. However, Python requires a standard library at runtime.
+# Therefore, it is necessary to add a build step to an Xcode app target that
+# processes the standard library and puts the content into the final app.
+#
+# In general, these tools will be invoked after bundle resources have been
+# copied into the app, but before framework embedding (and signing).
+#
+# The following is an example script, assuming that:
+# * Python.xcframework is in the root of the project
+# * There is an `app` folder that contains the app code
+# * There is an `app_packages` folder that contains installed Python packages.
+# -----
+#   set -e
+#   source $PROJECT_DIR/Python.xcframework/build/build_utils.sh
+#   install_python Python.xcframework app app_packages
+# -----
+
+# Copy the standard library from the XCframework into the app bundle.
+#
+# Accepts one argument:
+# 1. The path, relative to the root of the Xcode project, where the Python
+#    XCframework can be found.
+install_stdlib() {
+    PYTHON_XCFRAMEWORK_PATH=$1
+
+    mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib"
+    if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then
+        echo "Installing Python modules for iOS Simulator"
+        if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/ios-arm64-simulator" ]; then
+            SLICE_FOLDER="ios-arm64-simulator"
+        else
+            SLICE_FOLDER="ios-arm64_x86_64-simulator"
+        fi
+    else
+        echo "Installing Python modules for iOS Device"
+        SLICE_FOLDER="ios-arm64"
+    fi
+
+    # If the XCframework has a shared lib folder, then it's a full framework.
+    # Copy both the common and slice-specific part of the lib directory.
+    # Otherwise, it's a single-arch framework; use the "full" lib folder.
+    if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib" ]; then
+        rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/"
+        rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/"
+    else
+        # A single-arch framework will have a libpython symlink; that can't be included at runtime
+        rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib'
+    fi
+}
+
+# Convert a single .so library into a framework that iOS can load.
+#
+# Accepts three arguments:
+# 1. The path, relative to the root of the Xcode project, where the Python
+#    XCframework can be found.
+# 2. The base path, relative to the installed location in the app bundle, that
+#    needs to be processed. Any .so file found in this path (or a subdirectory
+#    of it) will be processed.
+# 2. The full path to a single .so file to process. This path should include
+#    the base path.
+install_dylib () {
+    PYTHON_XCFRAMEWORK_PATH=$1
+    INSTALL_BASE=$2
+    FULL_EXT=$3
+
+    # The name of the extension file
+    EXT=$(basename "$FULL_EXT")
+    # The name and location of the module
+    MODULE_PATH=$(dirname "$FULL_EXT")
+    MODULE_NAME=$(echo $EXT | cut -d "." -f 1)
+    # The location of the extension file, relative to the bundle
+    RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/}
+    # The path to the extension file, relative to the install base
+    PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}
+    # The full dotted name of the extension module, constructed from the file path.
+    FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" ".");
+    # A bundle identifier; not actually used, but required by Xcode framework packaging
+    FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-")
+    # The name of the framework folder.
+    FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework"
+
+    # If the framework folder doesn't exist, create it.
+    if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then
+        echo "Creating framework for $RELATIVE_EXT"
+        mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER"
+        cp "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/build/$PLATFORM_FAMILY_NAME-dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist"
+        plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist"
+        plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist"
+    fi
+
+    echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME"
+    mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME"
+    # Create a placeholder .fwork file where the .so was
+    echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork
+    # Create a back reference to the .so file location in the framework
+    echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin"
+
+    # If the framework provides an xcprivacy file, install it.
+    if [ -e "$MODULE_PATH/$MODULE_NAME.xcprivacy" ]; then
+        echo "Installing XCPrivacy file for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME"
+        XCPRIVACY_FILE="$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/PrivacyInfo.xcprivacy"
+        if [ -e "$XCPRIVACY_FILE" ]; then
+          rm -rf "$XCPRIVACY_FILE"
+        fi
+        mv "$MODULE_PATH/$MODULE_NAME.xcprivacy" "$XCPRIVACY_FILE"
+    fi
+
+    echo "Signing framework as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..."
+    /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER"
+}
+
+# Process all the dynamic libraries in a path into Framework format.
+#
+# Accepts two arguments:
+# 1. The path, relative to the root of the Xcode project, where the Python
+#    XCframework can be found.
+# 2. The base path, relative to the installed location in the app bundle, that
+#    needs to be processed. Any .so file found in this path (or a subdirectory
+#    of it) will be processed.
+process_dylibs () {
+    PYTHON_XCFRAMEWORK_PATH=$1
+    LIB_PATH=$2
+    find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do
+        install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT"
+    done
+}
+
+# The entry point for post-processing a Python XCframework.
+#
+# Accepts 1 or more arguments:
+# 1. The path, relative to the root of the Xcode project, where the Python
+#    XCframework can be found. If the XCframework is in the root of the project,
+# 2+. The path of a package, relative to the root of the packaged app, that contains
+#     library content that should be processed for binary libraries.
+install_python() {
+    PYTHON_XCFRAMEWORK_PATH=$1
+    shift
+
+    install_stdlib $PYTHON_XCFRAMEWORK_PATH
+    PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib")
+    echo "Install Python $PYTHON_VER standard library extension modules..."
+    process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload
+
+    for package_path in $@; do
+        echo "Installing $package_path extension modules ..."
+        process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path
+    done
+}

Apple/testbed/Testbed.lldbinit

@@ -0,0 +1,4 @@
+process handle SIGINT -n true -p true -s false
+process handle SIGUSR1 -n true -p true -s false
+process handle SIGUSR2 -n true -p true -s false
+process handle SIGXFSZ -n true -p true -s false

Apple/testbed/TestbedTests/TestbedTests.m

@@ -1,11 +1,11 @@
 #import <XCTest/XCTest.h>
 #import <Python/Python.h>
 
-@interface iOSTestbedTests : XCTestCase
+@interface TestbedTests : XCTestCase
 
 @end
 
-@implementation iOSTestbedTests
+@implementation TestbedTests
 
 
 - (void)testPython {
@@ -15,6 +15,11 @@
     PyStatus status;
     PyPreConfig preconfig;
     PyConfig config;
+    PyObject *app_packages_path;
+    PyObject *method_args;
+    PyObject *result;
+    PyObject *site_module;
+    PyObject *site_addsitedir_attr;
     PyObject *sys_module;
     PyObject *sys_path_attr;
     NSArray *test_args;
@@ -30,19 +35,26 @@
     setenv("NO_COLOR", "1", true);
     setenv("PYTHON_COLORS", "0", true);
 
+    if (getenv("GITHUB_ACTIONS")) {
+        NSLog(@"Running in a GitHub Actions environment");
+    }
     // Arguments to pass into the test suite runner.
     // argv[0] must identify the process; any subsequent arg
     // will be handled as if it were an argument to `python -m test`
-    test_args = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"TestArgs"];
+    // The processInfo arguments contain the binary that is running,
+    // followed by the arguments defined in the test plan. This means:
+    //    run_module = test_args[1]
+    //    argv = ["Testbed"] + test_args[2:]
+    test_args = [[NSProcessInfo processInfo] arguments];
     if (test_args == NULL) {
         NSLog(@"Unable to identify test arguments.");
     }
-    argv = malloc(sizeof(char *) * ([test_args count] + 1));
-    argv[0] = "iOSTestbed";
-    for (int i = 1; i < [test_args count]; i++) {
-        argv[i] = [[test_args objectAtIndex:i] UTF8String];
+    NSLog(@"Test arguments: %@", test_args);
+    argv = malloc(sizeof(char *) * ([test_args count] - 1));
+    argv[0] = "Testbed";
+    for (int i = 1; i < [test_args count] - 1; i++) {
+        argv[i] = [[test_args objectAtIndex:i+1] UTF8String];
     }
-    NSLog(@"Test command: %@", test_args);
 
     // Generate an isolated Python configuration.
     NSLog(@"Configuring isolated Python...");
@@ -63,7 +75,7 @@
     // Ensure that signal handlers are installed
     config.install_signal_handlers = 1;
     // Run the test module.
-    config.run_module = Py_DecodeLocale([[test_args objectAtIndex:0] UTF8String], NULL);
+    config.run_module = Py_DecodeLocale([[test_args objectAtIndex:1] UTF8String], NULL);
     // For debugging - enable verbose mode.
     // config.verbose = 1;
 
@@ -96,7 +108,7 @@
     }
 
     NSLog(@"Configure argc/argv...");
-    status = PyConfig_SetBytesArgv(&config, [test_args count], (char**) argv);
+    status = PyConfig_SetBytesArgv(&config, [test_args count] - 1, (char**) argv);
     if (PyStatus_Exception(status)) {
         XCTFail(@"Unable to configure argc/argv: %s", status.err_msg);
         PyConfig_Clear(&config);
@@ -111,6 +123,43 @@
         return;
     }
 
+    // Add app_packages as a site directory. This both adds to sys.path,
+    // and ensures that any .pth files in that directory will be executed.
+    site_module = PyImport_ImportModule("site");
+    if (site_module == NULL) {
+        XCTFail(@"Could not import site module");
+        return;
+    }
+
+    site_addsitedir_attr = PyObject_GetAttrString(site_module, "addsitedir");
+    if (site_addsitedir_attr == NULL || !PyCallable_Check(site_addsitedir_attr)) {
+        XCTFail(@"Could not access site.addsitedir");
+        return;
+    }
+
+    path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil];
+    NSLog(@"App packages path: %@", path);
+    wtmp_str = Py_DecodeLocale([path UTF8String], NULL);
+    app_packages_path = PyUnicode_FromWideChar(wtmp_str, wcslen(wtmp_str));
+    if (app_packages_path == NULL) {
+        XCTFail(@"Could not convert app_packages path to unicode");
+        return;
+    }
+    PyMem_RawFree(wtmp_str);
+
+    method_args = Py_BuildValue("(O)", app_packages_path);
+    if (method_args == NULL) {
+        XCTFail(@"Could not create arguments for site.addsitedir");
+        return;
+    }
+
+    result = PyObject_CallObject(site_addsitedir_attr, method_args);
+    if (result == NULL) {
+        XCTFail(@"Could not add app_packages directory using site.addsitedir");
+        return;
+    }
+
+    // Add test code to sys.path
     sys_module = PyImport_ImportModule("sys");
     if (sys_module == NULL) {
         XCTFail(@"Could not import sys module");
@@ -123,17 +172,6 @@
         return;
     }
 
-    // Add the app packages path
-    path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil];
-    NSLog(@"App packages path: %@", path);
-    wtmp_str = Py_DecodeLocale([path UTF8String], NULL);
-    failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String]));
-    if (failed) {
-        XCTFail(@"Unable to add app packages to sys.path");
-        return;
-    }
-    PyMem_RawFree(wtmp_str);
-
     path = [NSString stringWithFormat:@"%@/app", resourcePath, nil];
     NSLog(@"App path: %@", path);
     wtmp_str = Py_DecodeLocale([path UTF8String], NULL);

Apple/testbed/__main__.py

@@ -0,0 +1,435 @@
+import argparse
+import json
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+TEST_SLICES = {
+    "iOS": "ios-arm64_x86_64-simulator",
+}
+
+DECODE_ARGS = ("UTF-8", "backslashreplace")
+
+# The system log prefixes each line:
+#   2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ...
+#   2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ...
+
+LOG_PREFIX_REGEX = re.compile(
+    r"^\d{4}-\d{2}-\d{2}"  # YYYY-MM-DD
+    r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}"  # HH:MM:SS.ssssss+ZZZZ
+    r"\s+iOSTestbed\[\d+:\w+\]"  # Process/thread ID
+)
+
+
+# Select a simulator device to use.
+def select_simulator_device(platform):
+    # List the testing simulators, in JSON format
+    raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"])
+    json_data = json.loads(raw_json)
+
+    if platform == "iOS":
+        # Any iOS device will do; we'll look for "SE" devices - but the name
+        # isn't consistent over time. Older Xcode versions will use "iPhone SE
+        # (Nth generation)"; As of 2025, they've started using "iPhone 16e".
+        #
+        # When Xcode is updated after a new release, new devices will be
+        # available and old ones will be dropped from the set available on the
+        # latest iOS version. Select the one with the highest minimum runtime
+        # version - this is an indicator of the "newest" released device, which
+        # should always be supported on the "most recent" iOS version.
+        se_simulators = sorted(
+            (devicetype["minRuntimeVersion"], devicetype["name"])
+            for devicetype in json_data["devicetypes"]
+            if devicetype["productFamily"] == "iPhone"
+            and (
+                (
+                    "iPhone " in devicetype["name"]
+                    and devicetype["name"].endswith("e")
+                )
+                or "iPhone SE " in devicetype["name"]
+            )
+        )
+        simulator = se_simulators[-1][1]
+    else:
+        raise ValueError(f"Unknown platform {platform}")
+
+    return simulator
+
+
+def xcode_test(location: Path, platform: str, simulator: str, verbose: bool):
+    # Build and run the test suite on the named simulator.
+    args = [
+        "-project",
+        str(location / f"{platform}Testbed.xcodeproj"),
+        "-scheme",
+        f"{platform}Testbed",
+        "-destination",
+        f"platform={platform} Simulator,name={simulator}",
+        "-derivedDataPath",
+        str(location / "DerivedData"),
+    ]
+    verbosity_args = [] if verbose else ["-quiet"]
+
+    print("Building test project...")
+    subprocess.run(
+        ["xcodebuild", "build-for-testing"] + args + verbosity_args,
+        check=True,
+    )
+
+    # Any environment variable prefixed with TEST_RUNNER_ is exposed into the
+    # test runner environment. There are some variables (like those identifying
+    # CI platforms) that can be useful to have access to.
+    test_env = os.environ.copy()
+    if "GITHUB_ACTIONS" in os.environ:
+        test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"]
+
+    print("Running test project...")
+    # Test execution *can't* be run -quiet; verbose mode
+    # is how we see the output of the test output.
+    process = subprocess.Popen(
+        ["xcodebuild", "test-without-building"] + args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        env=test_env,
+    )
+    while line := (process.stdout.readline()).decode(*DECODE_ARGS):
+        # Strip the timestamp/process prefix from each log line
+        line = LOG_PREFIX_REGEX.sub("", line)
+        sys.stdout.write(line)
+        sys.stdout.flush()
+
+    status = process.wait(timeout=5)
+    exit(status)
+
+
+def copy(src, tgt):
+    """An all-purpose copy.
+
+    If src is a file, it is copied. If src is a symlink, it is copied *as a
+    symlink*. If src is a directory, the full tree is duplicated, with symlinks
+    being preserved.
+    """
+    if src.is_file() or src.is_symlink():
+        shutil.copyfile(src, tgt, follow_symlinks=False)
+    else:
+        shutil.copytree(src, tgt, symlinks=True)
+
+
+def clone_testbed(
+    source: Path,
+    target: Path,
+    framework: Path,
+    platform: str,
+    apps: list[Path],
+) -> None:
+    if target.exists():
+        print(f"{target} already exists; aborting without creating project.")
+        sys.exit(10)
+
+    if framework is None:
+        if not (
+            source / "Python.xcframework" / TEST_SLICES[platform] / "bin"
+        ).is_dir():
+            print(
+                f"The testbed being cloned ({source}) does not contain "
+                "a framework with slices. Re-run with --framework"
+            )
+            sys.exit(11)
+    else:
+        if not framework.is_dir():
+            print(f"{framework} does not exist.")
+            sys.exit(12)
+        elif not (
+            framework.suffix == ".xcframework"
+            or (framework / "Python.framework").is_dir()
+        ):
+            print(
+                f"{framework} is not an XCframework, "
+                f"or a simulator slice of a framework build."
+            )
+            sys.exit(13)
+
+    print("Cloning testbed project:")
+    print(f"  Cloning {source}...", end="")
+    # Only copy the files for the platform being cloned plus the files common
+    # to all platforms. The XCframework will be copied later, if needed.
+    target.mkdir(parents=True)
+
+    for name in [
+        "__main__.py",
+        "TestbedTests",
+        "Testbed.lldbinit",
+        f"{platform}Testbed",
+        f"{platform}Testbed.xcodeproj",
+        f"{platform}Testbed.xctestplan",
+    ]:
+        copy(source / name, target / name)
+
+    print(" done")
+
+    orig_xc_framework_path = source / "Python.xcframework"
+    xc_framework_path = target / "Python.xcframework"
+    test_framework_path = xc_framework_path / TEST_SLICES[platform]
+    if framework is not None:
+        if framework.suffix == ".xcframework":
+            print("  Installing XCFramework...", end="")
+            xc_framework_path.symlink_to(
+                framework.relative_to(xc_framework_path.parent, walk_up=True)
+            )
+            print(" done")
+        else:
+            print("  Installing simulator framework...", end="")
+            # We're only installing a slice of a framework; we need
+            # to do a full tree copy to make sure we don't damage
+            # symlinked content.
+            shutil.copytree(orig_xc_framework_path, xc_framework_path)
+            if test_framework_path.is_dir():
+                shutil.rmtree(test_framework_path)
+            else:
+                test_framework_path.unlink(missing_ok=True)
+            test_framework_path.symlink_to(
+                framework.relative_to(test_framework_path.parent, walk_up=True)
+            )
+            print(" done")
+    else:
+        copy(orig_xc_framework_path, xc_framework_path)
+
+        if (
+            xc_framework_path.is_symlink()
+            and not xc_framework_path.readlink().is_absolute()
+        ):
+            # XCFramework is a relative symlink. Rewrite the symlink relative
+            # to the new location.
+            print("  Rewriting symlink to XCframework...", end="")
+            resolved_xc_framework_path = (
+                source / xc_framework_path.readlink()
+            ).resolve()
+            xc_framework_path.unlink()
+            xc_framework_path.symlink_to(
+                resolved_xc_framework_path.relative_to(
+                    xc_framework_path.parent, walk_up=True
+                )
+            )
+            print(" done")
+        elif (
+            test_framework_path.is_symlink()
+            and not test_framework_path.readlink().is_absolute()
+        ):
+            print("  Rewriting symlink to simulator framework...", end="")
+            # Simulator framework is a relative symlink. Rewrite the symlink
+            # relative to the new location.
+            orig_test_framework_path = (
+                source / "Python.XCframework" / test_framework_path.readlink()
+            ).resolve()
+            test_framework_path.unlink()
+            test_framework_path.symlink_to(
+                orig_test_framework_path.relative_to(
+                    test_framework_path.parent, walk_up=True
+                )
+            )
+            print(" done")
+        else:
+            print("  Using pre-existing Python framework.")
+
+    for app_src in apps:
+        print(f"  Installing app {app_src.name!r}...", end="")
+        app_target = target / f"Testbed/app/{app_src.name}"
+        if app_target.is_dir():
+            shutil.rmtree(app_target)
+        shutil.copytree(app_src, app_target)
+        print(" done")
+
+    print(f"Successfully cloned testbed: {target.resolve()}")
+
+
+def update_test_plan(testbed_path, platform, args):
+    # Modify the test plan to use the requested test arguments.
+    test_plan_path = testbed_path / f"{platform}Testbed.xctestplan"
+    with test_plan_path.open("r", encoding="utf-8") as f:
+        test_plan = json.load(f)
+
+    test_plan["defaultOptions"]["commandLineArgumentEntries"] = [
+        {"argument": shlex.quote(arg)} for arg in args
+    ]
+
+    with test_plan_path.open("w", encoding="utf-8") as f:
+        json.dump(test_plan, f, indent=2)
+
+
+def run_testbed(
+    platform: str,
+    simulator: str | None,
+    args: list[str],
+    verbose: bool = False,
+):
+    location = Path(__file__).parent
+    print("Updating test plan...", end="")
+    update_test_plan(location, platform, args)
+    print(" done.")
+
+    if simulator is None:
+        simulator = select_simulator_device(platform)
+    print(f"Running test on {simulator}")
+
+    xcode_test(
+        location,
+        platform=platform,
+        simulator=simulator,
+        verbose=verbose,
+    )
+
+
+def main():
+    # Look for directories like `iOSTestbed` as an indicator of the platforms
+    # that the testbed folder supports. The original source testbed can support
+    # many platforms, but when cloned, only one platform is preserved.
+    available_platforms = [
+        platform
+        for platform in ["iOS"]
+        if (Path(__file__).parent / f"{platform}Testbed").is_dir()
+    ]
+
+    parser = argparse.ArgumentParser(
+        description=(
+            "Manages the process of testing an Apple Python project "
+            "through Xcode."
+        ),
+    )
+
+    subcommands = parser.add_subparsers(dest="subcommand")
+    clone = subcommands.add_parser(
+        "clone",
+        description=(
+            "Clone the testbed project, copying in a Python framework and"
+            "any specified application code."
+        ),
+        help="Clone a testbed project to a new location.",
+    )
+    clone.add_argument(
+        "--framework",
+        help=(
+            "The location of the XCFramework (or simulator-only slice of an "
+            "XCFramework) to use when running the testbed"
+        ),
+    )
+    clone.add_argument(
+        "--platform",
+        dest="platform",
+        choices=available_platforms,
+        default=available_platforms[0],
+        help=f"The platform to target (default: {available_platforms[0]})",
+    )
+    clone.add_argument(
+        "--app",
+        dest="apps",
+        action="append",
+        default=[],
+        help="The location of any code to include in the testbed project",
+    )
+    clone.add_argument(
+        "location",
+        help="The path where the testbed will be cloned.",
+    )
+
+    run = subcommands.add_parser(
+        "run",
+        usage=(
+            "%(prog)s [-h] [--simulator SIMULATOR] -- "
+            "<test arg> [<test arg> ...]"
+        ),
+        description=(
+            "Run a testbed project. The arguments provided after `--` will be "
+            "passed to the running iOS process as if they were arguments to "
+            "`python -m`."
+        ),
+        help="Run a testbed project",
+    )
+    run.add_argument(
+        "--platform",
+        dest="platform",
+        choices=available_platforms,
+        default=available_platforms[0],
+        help=f"The platform to target (default: {available_platforms[0]})",
+    )
+    run.add_argument(
+        "--simulator",
+        help=(
+            "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to "
+            "the most recently released 'entry level' iPhone device. Device "
+            "architecture and OS version can also be specified; e.g., "
+            "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on "
+            "an ARM64 iPhone 16 Pro simulator running iOS 26.0."
+        ),
+    )
+    run.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Enable verbose output",
+    )
+
+    try:
+        pos = sys.argv.index("--")
+        testbed_args = sys.argv[1:pos]
+        test_args = sys.argv[pos + 1 :]
+    except ValueError:
+        testbed_args = sys.argv[1:]
+        test_args = []
+
+    context = parser.parse_args(testbed_args)
+
+    if context.subcommand == "clone":
+        clone_testbed(
+            source=Path(__file__).parent.resolve(),
+            target=Path(context.location).resolve(),
+            framework=Path(context.framework).resolve()
+            if context.framework
+            else None,
+            platform=context.platform,
+            apps=[Path(app) for app in context.apps],
+        )
+    elif context.subcommand == "run":
+        if test_args:
+            if not (
+                Path(__file__).parent
+                / "Python.xcframework"
+                / TEST_SLICES[context.platform]
+                / "bin"
+            ).is_dir():
+                print(
+                    "Testbed does not contain a compiled Python framework. "
+                    f"Use `python {sys.argv[0]} clone ...` to create a "
+                    "runnable clone of this testbed."
+                )
+                sys.exit(20)
+
+            run_testbed(
+                platform=context.platform,
+                simulator=context.simulator,
+                verbose=context.verbose,
+                args=test_args,
+            )
+        else:
+            print(
+                "Must specify test arguments "
+                f"(e.g., {sys.argv[0]} run -- test)"
+            )
+            print()
+            parser.print_help(sys.stderr)
+            sys.exit(21)
+    else:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    # Under the buildbot, stdout is not a TTY, but we must still flush after
+    # every line to make sure our output appears in the correct order relative
+    # to the output of our subprocesses.
+    for stream in [sys.stdout, sys.stderr]:
+        stream.reconfigure(line_buffering=True)
+    main()

Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj

@@ -11,12 +11,11 @@
 		607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; };
 		607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; };
 		607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; };
-		607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; };
+		607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */; };
 		607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; };
 		607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; };
 		607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
-		607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; };
 		608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; };
 		608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; };
 /* End PBXBuildFile section */
@@ -64,12 +63,12 @@
 		607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
 		607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = "<group>"; };
 		607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
-		607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = "<group>"; };
+		607A66312B0EFA3A0010BFC8 /* TestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = TestbedTests.m; sourceTree = "<group>"; };
 		607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = "<group>"; };
-		607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = "<group>"; };
 		607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = "<group>"; };
 		608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = "<group>"; };
 		608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = "<group>"; };
+		60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = iOSTestbed.xctestplan; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -95,9 +94,10 @@
 		607A66092B0EFA380010BFC8 = {
 			isa = PBXGroup;
 			children = (
+				60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */,
 				607A664A2B0EFB310010BFC8 /* Python.xcframework */,
 				607A66142B0EFA380010BFC8 /* iOSTestbed */,
-				607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */,
+				607A66302B0EFA3A0010BFC8 /* TestbedTests */,
 				607A66132B0EFA380010BFC8 /* Products */,
 				607A664F2B0EFFE00010BFC8 /* Frameworks */,
 			);
@@ -118,7 +118,6 @@
 				608619552CB7819B00F46182 /* app */,
 				608619532CB77BA900F46182 /* app_packages */,
 				607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */,
-				607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */,
 				607A66152B0EFA380010BFC8 /* AppDelegate.h */,
 				607A66162B0EFA380010BFC8 /* AppDelegate.m */,
 				607A66212B0EFA390010BFC8 /* Assets.xcassets */,
@@ -128,12 +127,12 @@
 			path = iOSTestbed;
 			sourceTree = "<group>";
 		};
-		607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = {
+		607A66302B0EFA3A0010BFC8 /* TestbedTests */ = {
 			isa = PBXGroup;
 			children = (
-				607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */,
+				607A66312B0EFA3A0010BFC8 /* TestbedTests.m */,
 			);
-			path = iOSTestbedTests;
+			path = TestbedTests;
 			sourceTree = "<group>";
 		};
 		607A664F2B0EFFE00010BFC8 /* Frameworks */ = {
@@ -153,8 +152,7 @@
 				607A660E2B0EFA380010BFC8 /* Sources */,
 				607A660F2B0EFA380010BFC8 /* Frameworks */,
 				607A66102B0EFA380010BFC8 /* Resources */,
-				607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */,
-				607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */,
+				607A66552B0F061D0010BFC8 /* Process Python libraries */,
 				607A664E2B0EFC080010BFC8 /* Embed Frameworks */,
 			);
 			buildRules = (
@@ -228,7 +226,6 @@
 			buildActionMask = 2147483647;
 			files = (
 				607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */,
-				607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */,
 				608619562CB7819B00F46182 /* app in Resources */,
 				607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */,
 				608619542CB77BA900F46182 /* app_packages in Resources */,
@@ -245,7 +242,7 @@
 /* End PBXResourcesBuildPhase section */
 
 /* Begin PBXShellScriptBuildPhase section */
-		607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = {
+		607A66552B0F061D0010BFC8 /* Process Python libraries */ = {
 			isa = PBXShellScriptBuildPhase;
 			alwaysOutOfDate = 1;
 			buildActionMask = 2147483647;
@@ -255,34 +252,14 @@
 			);
 			inputPaths = (
 			);
-			name = "Install Target Specific Python Standard Library";
+			name = "Process Python libraries";
 			outputFileListPaths = (
 			);
 			outputPaths = (
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n    echo \"Installing Python modules for iOS Simulator\"\n    rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n    echo \"Installing Python modules for iOS Device\"\n    rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n";
-			showEnvVarsInLog = 0;
-		};
-		607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = {
-			isa = PBXShellScriptBuildPhase;
-			alwaysOutOfDate = 1;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			inputFileListPaths = (
-			);
-			inputPaths = (
-			);
-			name = "Prepare Python Binary Modules";
-			outputFileListPaths = (
-			);
-			outputPaths = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-			shellPath = /bin/sh;
-			shellScript = "set -e\n\ninstall_dylib () {\n    INSTALL_BASE=$1\n    FULL_EXT=$2\n\n    # The name of the extension file\n    EXT=$(basename \"$FULL_EXT\")\n    # The location of the extension file, relative to the bundle\n    RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n    # The path to the extension file, relative to the install base\n    PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n    # The full dotted name of the extension module, constructed from the file path.\n    FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n    # A bundle identifier; not actually used, but required by Xcode framework packaging\n    FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n    # The name of the framework folder.\n    FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n    # If the framework folder doesn't exist, create it.\n    if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n        echo \"Creating framework for $RELATIVE_EXT\" \n        mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n        cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n        plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n        plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n    fi\n    \n    echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n    mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n    # Create a placeholder .fwork file where the .so was\n    echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n    # Create a back reference to the .so file location in the framework\n    echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\"             \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n    install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n    install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n    install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n";
+			shellScript = "set -e\nsource $PROJECT_DIR/Python.xcframework/build/utils.sh\ninstall_python Python.xcframework app app_packages\n";
 			showEnvVarsInLog = 0;
 		};
 /* End PBXShellScriptBuildPhase section */
@@ -301,7 +278,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */,
+				607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -379,7 +356,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
@@ -434,7 +411,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				MTL_FAST_MATH = YES;
@@ -460,7 +437,7 @@
 				INFOPLIST_KEY_UIMainStoryboardFile = Main;
 				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
 				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -491,7 +468,7 @@
 				INFOPLIST_KEY_UIMainStoryboardFile = Main;
 				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
 				INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -514,7 +491,7 @@
 				DEVELOPMENT_TEAM = 3HEZE76D99;
 				GENERATE_INFOPLIST_FILE = YES;
 				HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\"";
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				MARKETING_VERSION = 1.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests;
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -534,7 +511,7 @@
 				DEVELOPMENT_TEAM = 3HEZE76D99;
 				GENERATE_INFOPLIST_FILE = YES;
 				HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\"";
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 13.0;
 				MARKETING_VERSION = 1.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests;
 				PRODUCT_NAME = "$(TARGET_NAME)";

Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme

@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1640"
+   version = "1.7">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES"
+      buildArchitectures = "Automatic">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "607A66112B0EFA380010BFC8"
+               BuildableName = "iOSTestbed.app"
+               BlueprintName = "iOSTestbed"
+               ReferencedContainer = "container:iOSTestbed.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      customLLDBInitFile = "$(SOURCE_ROOT)/Testbed.lldbinit"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <TestPlans>
+         <TestPlanReference
+            reference = "container:iOSTestbed.xctestplan"
+            default = "YES">
+         </TestPlanReference>
+      </TestPlans>
+      <Testables>
+         <TestableReference
+            skipped = "NO"
+            parallelizable = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "607A662C2B0EFA3A0010BFC8"
+               BuildableName = "iOSTestbedTests.xctest"
+               BlueprintName = "iOSTestbedTests"
+               ReferencedContainer = "container:iOSTestbed.xcodeproj">
+            </BuildableReference>
+         </TestableReference>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "607A66112B0EFA380010BFC8"
+            BuildableName = "iOSTestbed.app"
+            BlueprintName = "iOSTestbed"
+            ReferencedContainer = "container:iOSTestbed.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "607A66112B0EFA380010BFC8"
+            BuildableName = "iOSTestbed.app"
+            BlueprintName = "iOSTestbed"
+            ReferencedContainer = "container:iOSTestbed.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

Apple/testbed/iOSTestbed.xctestplan

@@ -0,0 +1,46 @@
+{
+  "configurations" : [
+    {
+      "id" : "F5A95CE4-1ADE-4A6E-A0E1-CDBAE26DF0C5",
+      "name" : "Test Scheme Action",
+      "options" : {
+
+      }
+    }
+  ],
+  "defaultOptions" : {
+    "commandLineArgumentEntries" : [
+      {
+        "argument" : "test"
+      },
+      {
+        "argument" : "-uall"
+      },
+      {
+        "argument" : "--single-process"
+      },
+      {
+        "argument" : "--rerun"
+      },
+      {
+        "argument" : "-W"
+      }
+    ],
+    "targetForVariableExpansion" : {
+      "containerPath" : "container:iOSTestbed.xcodeproj",
+      "identifier" : "607A66112B0EFA380010BFC8",
+      "name" : "iOSTestbed"
+    }
+  },
+  "testTargets" : [
+    {
+      "parallelizable" : false,
+      "target" : {
+        "containerPath" : "container:iOSTestbed.xcodeproj",
+        "identifier" : "607A662C2B0EFA3A0010BFC8",
+        "name" : "iOSTestbedTests"
+      }
+    }
+  ],
+  "version" : 1
+}

Apple/testbed/iOSTestbed/app/README

@@ -1,7 +1,7 @@
 This folder can contain any Python application code.
 
 During the build, any binary modules found in this folder will be processed into
-iOS Framework form.
+Framework form.
 
 When the test suite runs, this folder will be on the PYTHONPATH, and will be the
 working directory for the test suite.

Apple/testbed/iOSTestbed/app_packages/README

@@ -2,6 +2,6 @@ This folder can be a target for installing any Python dependencies needed by the
 test suite.
 
 During the build, any binary modules found in this folder will be processed into
-iOS Framework form.
+Framework form.
 
 When the test suite runs, this folder will be on the PYTHONPATH.

Apple/testbed/iOSTestbed/iOSTestbed-Info.plist

@@ -41,18 +41,6 @@
 		<string>UIInterfaceOrientationLandscapeLeft</string>
 		<string>UIInterfaceOrientationLandscapeRight</string>
 	</array>
-	<key>TestArgs</key>
-	<array>
-		<string>test</string> <!-- Invoke "python -m test" -->
-        <string>-uall</string> <!-- Enable all resources -->
-        <string>--single-process</string> <!-- always run all tests sequentially in a single process -->
-        <string>--rerun</string> <!-- Re-run failed tests in verbose mode -->
-        <string>-W</string> <!-- Display test output on failure -->
-		<!-- To run a subset of tests, add the test names below; e.g.,
-        <string>test_os</string>
-        <string>test_sys</string>
-		-->
-    </array>
 	<key>UIApplicationSceneManifest</key>
 	<dict>
 		<key>UIApplicationSupportsMultipleScenes</key>

Doc/Makefile

@@ -170,6 +170,7 @@ venv:
 		echo "venv already exists."; \
 		echo "To recreate it, remove it first with \`make clean-venv'."; \
 	else \
+		set -e; \
 		echo "Creating venv in $(VENVDIR)"; \
 		if $(UV) --version >/dev/null 2>&1; then \
 			$(UV) venv --python=$(PYTHON) $(VENVDIR); \
@@ -183,7 +184,7 @@ venv:
 	fi
 
 .PHONY: dist-no-html
-dist-no-html: dist-text dist-pdf dist-epub dist-texinfo
+dist-no-html: dist-text dist-epub dist-texinfo
 
 .PHONY: dist
 dist:
@@ -240,7 +241,8 @@ dist-pdf:
 	# as otherwise the full latexmk process is run twice.
 	# ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References)
 	-sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile
-	(cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2)
+	if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \
+	(cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2)
 	cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip
 	cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2
 	@echo "Build finished and archived!"

Doc/bugs.rst

@@ -19,6 +19,12 @@ If you find a bug in this documentation or would like to propose an improvement,
 please submit a bug report on the :ref:`issue tracker <using-the-tracker>`.  If you
 have a suggestion on how to fix it, include that as well.
 
+.. only:: translation
+
+   If the bug or suggested improvement concerns the translation of this
+   documentation, submit the report to the
+   `translation’s repository <TRANSLATION_REPO_>`_ instead.
+
 You can also open a discussion item on our
 `Documentation Discourse forum <https://discuss.python.org/c/documentation/26>`_.
 

Doc/c-api/allocation.rst

@@ -16,7 +16,20 @@ Allocating Objects on the Heap
 
    Initialize a newly allocated object *op* with its type and initial
    reference.  Returns the initialized object.  Other fields of the object are
-   not affected.
+   not initialized.  Despite its name, this function is unrelated to the
+   object's :meth:`~object.__init__` method (:c:member:`~PyTypeObject.tp_init`
+   slot).  Specifically, this function does **not** call the object's
+   :meth:`!__init__` method.
+
+   In general, consider this function to be a low-level routine. Use
+   :c:member:`~PyTypeObject.tp_alloc` where possible.
+   For implementing :c:member:`!tp_alloc` for your type, prefer
+   :c:func:`PyType_GenericAlloc` or :c:func:`PyObject_New`.
+
+   .. note::
+
+      This function only initializes the object's memory corresponding to the
+      initial :c:type:`PyObject` structure.  It does not zero the rest.
 
 
 .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size)
@@ -24,43 +37,108 @@ Allocating Objects on the Heap
    This does everything :c:func:`PyObject_Init` does, and also initializes the
    length information for a variable-size object.
 
+   .. note::
+
+      This function only initializes some of the object's memory.  It does not
+      zero the rest.
+
 
 .. c:macro:: PyObject_New(TYPE, typeobj)
 
-   Allocate a new Python object using the C structure type *TYPE*
-   and the Python type object *typeobj* (``PyTypeObject*``).
-   Fields not defined by the Python object header are not initialized.
-   The caller will own the only reference to the object
-   (i.e. its reference count will be one).
-   The size of the memory allocation is determined from the
-   :c:member:`~PyTypeObject.tp_basicsize` field of the type object.
+   Allocates a new Python object using the C structure type *TYPE* and the
+   Python type object *typeobj* (``PyTypeObject*``) by calling
+   :c:func:`PyObject_Malloc` to allocate memory and initializing it like
+   :c:func:`PyObject_Init`.  The caller will own the only reference to the
+   object (i.e. its reference count will be one).
 
-   Note that this function is unsuitable if *typeobj* has
-   :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects,
-   use :c:func:`PyObject_GC_New` instead.
+   Avoid calling this directly to allocate memory for an object; call the type's
+   :c:member:`~PyTypeObject.tp_alloc` slot instead.
+
+   When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot,
+   :c:func:`PyType_GenericAlloc` is preferred over a custom function that
+   simply calls this macro.
+
+   This macro does not call :c:member:`~PyTypeObject.tp_alloc`,
+   :c:member:`~PyTypeObject.tp_new` (:meth:`~object.__new__`), or
+   :c:member:`~PyTypeObject.tp_init` (:meth:`~object.__init__`).
+
+   This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in
+   :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_New` instead.
+
+   Memory allocated by this macro must be freed with :c:func:`PyObject_Free`
+   (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot).
+
+   .. note::
+
+      The returned memory is not guaranteed to have been completely zeroed
+      before it was initialized.
+
+   .. note::
+
+      This macro does not construct a fully initialized object of the given
+      type; it merely allocates memory and prepares it for further
+      initialization by :c:member:`~PyTypeObject.tp_init`.  To construct a
+      fully initialized object, call *typeobj* instead.  For example::
+
+         PyObject *foo = PyObject_CallNoArgs((PyObject *)&PyFoo_Type);
+
+   .. seealso::
+
+      * :c:func:`PyObject_Free`
+      * :c:macro:`PyObject_GC_New`
+      * :c:func:`PyType_GenericAlloc`
+      * :c:member:`~PyTypeObject.tp_alloc`
 
 
 .. c:macro:: PyObject_NewVar(TYPE, typeobj, size)
 
-   Allocate a new Python object using the C structure type *TYPE* and the
-   Python type object *typeobj* (``PyTypeObject*``).
-   Fields not defined by the Python object header
-   are not initialized.  The allocated memory allows for the *TYPE* structure
-   plus *size* (``Py_ssize_t``) fields of the size
-   given by the :c:member:`~PyTypeObject.tp_itemsize` field of
-   *typeobj*.  This is useful for implementing objects like tuples, which are
-   able to determine their size at construction time.  Embedding the array of
-   fields into the same allocation decreases the number of allocations,
-   improving the memory management efficiency.
+   Like :c:macro:`PyObject_New` except:
 
-   Note that this function is unsuitable if *typeobj* has
-   :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects,
-   use :c:func:`PyObject_GC_NewVar` instead.
+   * It allocates enough memory for the *TYPE* structure plus *size*
+     (``Py_ssize_t``) fields of the size given by the
+     :c:member:`~PyTypeObject.tp_itemsize` field of *typeobj*.
+   * The memory is initialized like :c:func:`PyObject_InitVar`.
 
+   This is useful for implementing objects like tuples, which are able to
+   determine their size at construction time.  Embedding the array of fields
+   into the same allocation decreases the number of allocations, improving the
+   memory management efficiency.
 
-.. c:function:: void PyObject_Del(void *op)
+   Avoid calling this directly to allocate memory for an object; call the type's
+   :c:member:`~PyTypeObject.tp_alloc` slot instead.
+
+   When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot,
+   :c:func:`PyType_GenericAlloc` is preferred over a custom function that
+   simply calls this macro.
+
+   This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in
+   :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_NewVar`
+   instead.
+
+   Memory allocated by this function must be freed with :c:func:`PyObject_Free`
+   (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot).
+
+   .. note::
+
+      The returned memory is not guaranteed to have been completely zeroed
+      before it was initialized.
+
+   .. note::
+
+      This macro does not construct a fully initialized object of the given
+      type; it merely allocates memory and prepares it for further
+      initialization by :c:member:`~PyTypeObject.tp_init`.  To construct a
+      fully initialized object, call *typeobj* instead.  For example::
+
+         PyObject *list_instance = PyObject_CallNoArgs((PyObject *)&PyList_Type);
+
+   .. seealso::
+
+      * :c:func:`PyObject_Free`
+      * :c:macro:`PyObject_GC_NewVar`
+      * :c:func:`PyType_GenericAlloc`
+      * :c:member:`~PyTypeObject.tp_alloc`
 
-   Same as :c:func:`PyObject_Free`.
 
 .. c:var:: PyObject _Py_NoneStruct
 
@@ -71,6 +149,38 @@ Allocating Objects on the Heap
 
 .. seealso::
 
-   :c:func:`PyModule_Create`
+   :ref:`moduleobjects`
       To allocate and create extension modules.
 
+
+Deprecated aliases
+^^^^^^^^^^^^^^^^^^
+
+These are :term:`soft deprecated` aliases to existing functions and macros.
+They exist solely for backwards compatibility.
+
+
+.. list-table::
+   :widths: auto
+   :header-rows: 1
+
+   * * Deprecated alias
+     * Function
+   * * .. c:macro:: PyObject_NEW(type, typeobj)
+     * :c:macro:`PyObject_New`
+   * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n)
+     * :c:macro:`PyObject_NewVar`
+   * * .. c:macro:: PyObject_INIT(op, typeobj)
+     * :c:func:`PyObject_Init`
+   * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n)
+     * :c:func:`PyObject_InitVar`
+   * * .. c:macro:: PyObject_MALLOC(n)
+     * :c:func:`PyObject_Malloc`
+   * * .. c:macro:: PyObject_REALLOC(p, n)
+     * :c:func:`PyObject_Realloc`
+   * * .. c:macro:: PyObject_FREE(p)
+     * :c:func:`PyObject_Free`
+   * * .. c:macro:: PyObject_DEL(p)
+     * :c:func:`PyObject_Free`
+   * * .. c:macro:: PyObject_Del(p)
+     * :c:func:`PyObject_Free`

Doc/c-api/arg.rst

@@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C:
 ``z`` (:class:`str` or ``None``) [const char \*]
    Like ``s``, but the Python object may also be ``None``, in which case the C
    pointer is set to ``NULL``.
-   It is the same as ``s?`` with the C pointer was initialized to ``NULL``.
 
 ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer]
    Like ``s*``, but the Python object may also be ``None``, in which case the
    ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``.
-   It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer`
-   structure was initialized to ``NULL``.
 
 ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`]
    Like ``s#``, but the Python object may also be ``None``, in which case the C
    pointer is set to ``NULL``.
-   It is the same as ``s#?`` with the C pointer was initialized to ``NULL``.
 
 ``y`` (read-only :term:`bytes-like object`) [const char \*]
    This format converts a bytes-like object to a C pointer to a
@@ -164,7 +160,7 @@ There are three ways strings and buffers can be converted to C:
 ``w*`` (read-write :term:`bytes-like object`) [Py_buffer]
    This format accepts any object which implements the read-write buffer
    interface. It fills a :c:type:`Py_buffer` structure provided by the caller.
-   The buffer may contain embedded null bytes. The caller have to call
+   The buffer may contain embedded null bytes. The caller has to call
    :c:func:`PyBuffer_Release` when it is done with the buffer.
 
 ``es`` (:class:`str`) [const char \*encoding, char \*\*buffer]
@@ -387,17 +383,6 @@ Other objects
       Non-tuple sequences are deprecated if *items* contains format units
       which store a borrowed buffer or a borrowed reference.
 
-``unit?`` (anything or ``None``) [*matching-variable(s)*]
-   ``?`` modifies the behavior of the preceding format unit.
-   The C variable(s) corresponding to that parameter should be initialized
-   to their default value --- when the argument is ``None``,
-   :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding
-   C variable(s).
-   If the argument is not ``None``, it is parsed according to the specified
-   format unit.
-
-   .. versionadded:: 3.14
-
 A few other characters have a meaning in a format string.  These may not occur
 inside nested parentheses.  They are:
 
@@ -685,6 +670,13 @@ Building values
 
    ``p`` (:class:`bool`) [int]
       Convert a C :c:expr:`int` to a Python :class:`bool` object.
+
+      Be aware that this format requires an ``int`` argument.
+      Unlike most other contexts in C, variadic arguments are not coerced to
+      a suitable type automatically.
+      You can convert another type (for example, a pointer or a float) to a
+      suitable ``int`` value using ``(x) ? 1 : 0`` or ``!!x``.
+
       .. versionadded:: 3.14
 
    ``c`` (:class:`bytes` of length 1) [char]

Doc/c-api/buffer.rst

@@ -261,6 +261,10 @@ readonly, format
       MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE`
       can be used to request a simple writable buffer.
 
+   .. c:macro:: PyBUF_WRITEABLE
+
+      This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`.
+
    .. c:macro:: PyBUF_FORMAT
 
       Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST

Doc/c-api/bytes.rst

@@ -219,3 +219,38 @@ called with a non-bytes parameter.
    reallocation fails, the original bytes object at *\*bytes* is deallocated,
    *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is
    returned.
+
+
+.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes)
+
+   Get the string representation of *bytes*. This function is currently used to
+   implement :meth:`!bytes.__repr__` in Python.
+
+   This function does not do type checking; it is undefined behavior to pass
+   *bytes* as a non-bytes object or ``NULL``.
+
+   If *smartquotes* is true, the representation will use a double-quoted string
+   instead of single-quoted string when single-quotes are present in *bytes*.
+   For example, the byte string ``'Python'`` would be represented as
+   ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is
+   false.
+
+   On success, this function returns a :term:`strong reference` to a
+   :class:`str` object containing the representation. On failure, this
+   returns ``NULL`` with an exception set.
+
+
+.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding)
+
+   Unescape a backslash-escaped string *s*. *s* must not be ``NULL``.
+   *len* must be the size of *s*.
+
+   *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If
+   *errors* is ``NULL``, then ``"strict"`` is used by default.
+
+   On success, this function returns a :term:`strong reference` to a Python
+   :class:`bytes` object containing the unescaped string. On failure, this
+   function returns ``NULL`` with an exception set.
+
+   .. versionchanged:: 3.9
+      *unicode* and *recode_encoding* are now unused.

Doc/c-api/capsule.rst

@@ -15,13 +15,19 @@ Refer to :ref:`using-capsules` for more information on using these objects.
 .. c:type:: PyCapsule
 
    This subtype of :c:type:`PyObject` represents an opaque value, useful for C
-   extension modules who need to pass an opaque value (as a :c:expr:`void*`
+   extension modules which need to pass an opaque value (as a :c:expr:`void*`
    pointer) through Python code to other C code.  It is often used to make a C
    function pointer defined in one module available to other modules, so the
    regular import mechanism can be used to access C APIs defined in dynamically
    loaded modules.
 
 
+.. c:var:: PyTypeObject PyCapsule_Type
+
+   The type object corresponding to capsule objects. This is the same object
+   as :class:`types.CapsuleType` in the Python layer.
+
+
 .. c:type:: PyCapsule_Destructor
 
    The type of a destructor callback for a capsule.  Defined as::
@@ -105,9 +111,19 @@ Refer to :ref:`using-capsules` for more information on using these objects.
    ``module.attribute``.  The *name* stored in the capsule must match this
    string exactly.
 
+   This function splits *name* on the ``.`` character, and imports the first
+   element. It then processes further elements using attribute lookups.
+
    Return the capsule's internal *pointer* on success.  On failure, set an
    exception and return ``NULL``.
 
+   .. note::
+
+      If *name* points to an attribute of some submodule or subpackage, this
+      submodule or subpackage must be previously imported using other means
+      (for example, by using :c:func:`PyImport_ImportModule`) for the
+      attribute lookups to succeed.
+
    .. versionchanged:: 3.3
       *no_block* has no effect anymore.
 

Doc/c-api/cell.rst

@@ -7,7 +7,7 @@ Cell Objects
 
 "Cell" objects are used to implement variables referenced by multiple scopes.
 For each such variable, a cell object is created to store the value; the local
-variables of each stack frame that references the value contains a reference to
+variables of each stack frame that references the value contain a reference to
 the cells from outer scopes which also use that variable.  When the value is
 accessed, the value contained in the cell is used instead of the cell object
 itself.  This de-referencing of the cell object requires support from the

Doc/c-api/code.rst

@@ -182,7 +182,7 @@ bound into a function.
    Type of a code object watcher callback function.
 
    If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked
-   after `co` has been fully initialized. Otherwise, the callback is invoked
+   after *co* has been fully initialized. Otherwise, the callback is invoked
    before the destruction of *co* takes place, so the prior state of *co*
    can be inspected.
 
@@ -211,6 +211,82 @@ bound into a function.
    .. versionadded:: 3.12
 
 
+.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj)
+
+   This is a :term:`soft deprecated` function that does nothing.
+
+   Prior to Python 3.10, this function would perform basic optimizations to a
+   code object.
+
+   .. versionchanged:: 3.10
+      This function now does nothing.
+
+
+.. _c_codeobject_flags:
+
+Code Object Flags
+-----------------
+
+Code objects contain a bit-field of flags, which can be retrieved as the
+:attr:`~codeobject.co_flags` Python attribute (for example using
+:c:func:`PyObject_GetAttrString`), and set using a *flags* argument to
+:c:func:`PyUnstable_Code_New` and similar functions.
+
+Flags whose names start with ``CO_FUTURE_`` correspond to features normally
+selectable by :ref:`future statements <future>`. These flags can be used in
+:c:member:`PyCompilerFlags.cf_flags`.
+Note that many ``CO_FUTURE_`` flags are mandatory in current versions of
+Python, and setting them has no effect.
+
+The following flags are available.
+For their meaning, see the linked documentation of their Python equivalents.
+
+
+.. list-table::
+   :widths: auto
+   :header-rows: 1
+
+   * * Flag
+     * Meaning
+   * * .. c:macro:: CO_OPTIMIZED
+     * :py:data:`inspect.CO_OPTIMIZED`
+   * * .. c:macro:: CO_NEWLOCALS
+     * :py:data:`inspect.CO_NEWLOCALS`
+   * * .. c:macro:: CO_VARARGS
+     * :py:data:`inspect.CO_VARARGS`
+   * * .. c:macro:: CO_VARKEYWORDS
+     * :py:data:`inspect.CO_VARKEYWORDS`
+   * * .. c:macro:: CO_NESTED
+     * :py:data:`inspect.CO_NESTED`
+   * * .. c:macro:: CO_GENERATOR
+     * :py:data:`inspect.CO_GENERATOR`
+   * * .. c:macro:: CO_COROUTINE
+     * :py:data:`inspect.CO_COROUTINE`
+   * * .. c:macro:: CO_ITERABLE_COROUTINE
+     * :py:data:`inspect.CO_ITERABLE_COROUTINE`
+   * * .. c:macro:: CO_ASYNC_GENERATOR
+     * :py:data:`inspect.CO_ASYNC_GENERATOR`
+   * * .. c:macro:: CO_HAS_DOCSTRING
+     * :py:data:`inspect.CO_HAS_DOCSTRING`
+   * * .. c:macro:: CO_METHOD
+     * :py:data:`inspect.CO_METHOD`
+
+   * * .. c:macro:: CO_FUTURE_DIVISION
+     * no effect (:py:data:`__future__.division`)
+   * * .. c:macro:: CO_FUTURE_ABSOLUTE_IMPORT
+     * no effect (:py:data:`__future__.absolute_import`)
+   * * .. c:macro:: CO_FUTURE_WITH_STATEMENT
+     * no effect (:py:data:`__future__.with_statement`)
+   * * .. c:macro:: CO_FUTURE_PRINT_FUNCTION
+     * no effect (:py:data:`__future__.print_function`)
+   * * .. c:macro:: CO_FUTURE_UNICODE_LITERALS
+     * no effect (:py:data:`__future__.unicode_literals`)
+   * * .. c:macro:: CO_FUTURE_GENERATOR_STOP
+     * no effect (:py:data:`__future__.generator_stop`)
+   * * .. c:macro:: CO_FUTURE_ANNOTATIONS
+     * :py:data:`__future__.annotations`
+
+
 Extra information
 -----------------
 
@@ -224,7 +300,7 @@ may change without deprecation warnings.
 
 .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free)
 
-   Return a new an opaque index value used to adding data to code objects.
+   Return a new opaque index value used to adding data to code objects.
 
    You generally call this function once (per interpreter) and use the result
    with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate

Doc/c-api/codec.rst

@@ -7,7 +7,7 @@ Codec registry and support functions
 
    Register a new codec search function.
 
-   As side effect, this tries to load the :mod:`!encodings` package, if not yet
+   As a side effect, this tries to load the :mod:`!encodings` package, if not yet
    done, to make sure that it is always first in the list of search functions.
 
 .. c:function:: int PyCodec_Unregister(PyObject *search_function)
@@ -39,7 +39,7 @@ Codec registry and support functions
    *object* is passed through the decoder function found for the given
    *encoding* using the error handling method defined by *errors*.  *errors* may
    be ``NULL`` to use the default method defined for the codec.  Raises a
-   :exc:`LookupError` if no encoder can be found.
+   :exc:`LookupError` if no decoder can be found.
 
 
 Codec lookup API
@@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers
    Replace the unicode encode error with ``\N{...}`` escapes.
 
    .. versionadded:: 3.5
+
+
+Codec utility variables
+-----------------------
+
+.. c:var:: const char *Py_hexdigits
+
+   A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``.
+
+   .. versionadded:: 3.3

Doc/c-api/concrete.rst

@@ -109,11 +109,20 @@ Other Objects
    descriptor.rst
    slice.rst
    memoryview.rst
+   picklebuffer.rst
    weakref.rst
    capsule.rst
    frame.rst
    gen.rst
    coro.rst
    contextvars.rst
-   datetime.rst
    typehints.rst
+
+
+C API for extension modules
+===========================
+
+.. toctree::
+
+   curses.rst
+   datetime.rst

Doc/c-api/conversion.rst

@@ -41,7 +41,7 @@ The return value (*rv*) for these functions should be interpreted as follows:
   ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'``
   in this case.
 
-* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in
+* When ``rv < 0``, the output conversion failed and ``str[size-1]`` is ``'\0'`` in
   this case too, but the rest of *str* is undefined. The exact cause of the error
   depends on the underlying platform.
 
@@ -128,18 +128,28 @@ The following functions provide locale-independent string to number conversions.
    must be 0 and is ignored.  The ``'r'`` format code specifies the
    standard :func:`repr` format.
 
-   *flags* can be zero or more of the values ``Py_DTSF_SIGN``,
-   ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together:
+   *flags* can be zero or more of the following values or-ed together:
 
-   * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign
-     character, even if *val* is non-negative.
+   .. c:macro:: Py_DTSF_SIGN
 
-   * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look
-     like an integer.
+      Always precede the returned string with a sign
+      character, even if *val* is non-negative.
 
-   * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules.  See the
-     documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for
-     details.
+   .. c:macro:: Py_DTSF_ADD_DOT_0
+
+      Ensure that the returned string will not look like an integer.
+
+   .. c:macro:: Py_DTSF_ALT
+
+      Apply "alternate" formatting rules.
+      See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for
+      details.
+
+   .. c:macro:: Py_DTSF_NO_NEG_0
+
+      Negative zero is converted to positive zero.
+
+      .. versionadded:: 3.11
 
    If *ptype* is non-``NULL``, then the value it points to will be set to one of
    ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that
@@ -152,13 +162,85 @@ The following functions provide locale-independent string to number conversions.
    .. versionadded:: 3.1
 
 
-.. c:function:: int PyOS_stricmp(const char *s1, const char *s2)
+.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2)
+                int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size)
 
-   Case insensitive comparison of strings. The function works almost
-   identically to :c:func:`!strcmp` except that it ignores the case.
+   Case insensitive comparison of strings. These functions work almost
+   identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively),
+   except that they ignore the case of ASCII characters.
+
+   Return ``0`` if the strings are equal, a negative value if *str1* sorts
+   lexicographically before *str2*, or a positive value if it sorts after.
+
+   In the *str1* or *str2* arguments, a NUL byte marks the end of the string.
+   For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size
+   of the string, as if NUL was present at the index given by *size*.
+
+   These functions do not use the locale.
 
 
-.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t  size)
+.. c:function:: int PyOS_stricmp(const char *str1, const char *str2)
+                int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t  size)
 
-   Case insensitive comparison of strings. The function works almost
-   identically to :c:func:`!strncmp` except that it ignores the case.
+   Case insensitive comparison of strings.
+
+   On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`,
+   respectively.
+
+   On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and
+   :c:func:`PyOS_mystrnicmp`, respectively.
+
+
+Character classification and conversion
+=======================================
+
+The following macros provide locale-independent (unlike the C standard library
+``ctype.h``) character classification and conversion.
+The argument must be a signed or unsigned :c:expr:`char`.
+
+
+.. c:macro:: Py_ISALNUM(c)
+
+   Return true if the character *c* is an alphanumeric character.
+
+
+.. c:macro:: Py_ISALPHA(c)
+
+   Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``).
+
+
+.. c:macro:: Py_ISDIGIT(c)
+
+   Return true if the character *c* is a decimal digit (``0-9``).
+
+
+.. c:macro:: Py_ISLOWER(c)
+
+   Return true if the character *c* is a lowercase ASCII letter (``a-z``).
+
+
+.. c:macro:: Py_ISUPPER(c)
+
+   Return true if the character *c* is an uppercase ASCII letter (``A-Z``).
+
+
+.. c:macro:: Py_ISSPACE(c)
+
+   Return true if the character *c* is a whitespace character (space, tab,
+   carriage return, newline, vertical tab, or form feed).
+
+
+.. c:macro:: Py_ISXDIGIT(c)
+
+   Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and
+   ``A-F``).
+
+
+.. c:macro:: Py_TOLOWER(c)
+
+   Return the lowercase equivalent of the character *c*.
+
+
+.. c:macro:: Py_TOUPPER(c)
+
+   Return the uppercase equivalent of the character *c*.

Doc/c-api/curses.rst

@@ -0,0 +1,138 @@
+.. highlight:: c
+
+Curses C API
+------------
+
+:mod:`curses` exposes a small C interface for extension modules.
+Consumers must include the header file :file:`py_curses.h` (which is not
+included by default by :file:`Python.h`) and :c:func:`import_curses` must
+be invoked, usually as part of the module initialisation function, to populate
+:c:var:`PyCurses_API`.
+
+.. warning::
+
+   Neither the C API nor the pure Python :mod:`curses` module are compatible
+   with subinterpreters.
+
+.. c:macro:: import_curses()
+
+   Import the curses C API. The macro does not need a semi-colon to be called.
+
+   On success, populate the :c:var:`PyCurses_API` pointer.
+
+   On failure, set :c:var:`PyCurses_API` to NULL and set an exception.
+   The caller must check if an error occurred via :c:func:`PyErr_Occurred`:
+
+   .. code-block::
+
+      import_curses();  // semi-colon is optional but recommended
+      if (PyErr_Occurred()) { /* cleanup */ }
+
+
+.. c:var:: void **PyCurses_API
+
+   Dynamically allocated object containing the curses C API.
+   This variable is only available once :c:macro:`import_curses` succeeds.
+
+   ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`.
+
+   ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]``
+   are pointers to predicate functions of type ``int (*)(void)``.
+
+   When called, these predicates return whether :func:`curses.setupterm`,
+   :func:`curses.initscr`, and :func:`curses.start_color` have been called
+   respectively.
+
+   See also the convenience macros :c:macro:`PyCursesSetupTermCalled`,
+   :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`.
+
+   .. note::
+
+      The number of entries in this structure is subject to changes.
+      Consider using :c:macro:`PyCurses_API_pointers` to check if
+      new fields are available or not.
+
+
+.. c:macro:: PyCurses_API_pointers
+
+   The number of accessible fields (``4``) in :c:var:`PyCurses_API`.
+   This number is incremented whenever new fields are added.
+
+
+.. c:var:: PyTypeObject PyCursesWindow_Type
+
+   The :ref:`heap type <heap-types>` corresponding to :class:`curses.window`.
+
+
+.. c:function:: int PyCursesWindow_Check(PyObject *op)
+
+   Return true if *op* is a :class:`curses.window` instance, false otherwise.
+
+
+The following macros are convenience macros expanding into C statements.
+In particular, they can only be used as ``macro;`` or ``macro``, but not
+``macro()`` or ``macro();``.
+
+.. c:macro:: PyCursesSetupTermCalled
+
+   Macro checking if :func:`curses.setupterm` has been called.
+
+   The macro expansion is roughly equivalent to:
+
+   .. code-block::
+
+      {
+          typedef int (*predicate_t)(void);
+          predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1];
+          if (!was_setupterm_called()) {
+              return NULL;
+          }
+      }
+
+
+.. c:macro:: PyCursesInitialised
+
+   Macro checking if :func:`curses.initscr` has been called.
+
+   The macro expansion is roughly equivalent to:
+
+   .. code-block::
+
+      {
+          typedef int (*predicate_t)(void);
+          predicate_t was_initscr_called = (predicate_t)PyCurses_API[2];
+          if (!was_initscr_called()) {
+              return NULL;
+          }
+      }
+
+
+.. c:macro:: PyCursesInitialisedColor
+
+   Macro checking if :func:`curses.start_color` has been called.
+
+   The macro expansion is roughly equivalent to:
+
+   .. code-block::
+
+      {
+          typedef int (*predicate_t)(void);
+          predicate_t was_start_color_called = (predicate_t)PyCurses_API[3];
+          if (!was_start_color_called()) {
+              return NULL;
+          }
+      }
+
+
+Internal data
+-------------
+
+The following objects are exposed by the C API but should be considered
+internal-only.
+
+.. c:macro:: PyCurses_CAPSULE_NAME
+
+   Name of the curses capsule to pass to :c:func:`PyCapsule_Import`.
+
+   Internal usage only. Use :c:macro:`import_curses` instead.
+

Doc/c-api/datetime.rst

@@ -8,11 +8,42 @@ DateTime Objects
 Various date and time objects are supplied by the :mod:`datetime` module.
 Before using any of these functions, the header file :file:`datetime.h` must be
 included in your source (note that this is not included by :file:`Python.h`),
-and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of
+and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of
 the module initialisation function.  The macro puts a pointer to a C structure
-into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following
+into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following
 macros.
 
+.. c:macro:: PyDateTime_IMPORT()
+
+   Import the datetime C API.
+
+   On success, populate the :c:var:`PyDateTimeAPI` pointer.
+   On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception.
+   The caller must check if an error occurred via :c:func:`PyErr_Occurred`:
+
+   .. code-block::
+
+      PyDateTime_IMPORT;
+      if (PyErr_Occurred()) { /* cleanup */ }
+
+   .. warning::
+
+      This is not compatible with subinterpreters.
+
+.. c:type:: PyDateTime_CAPI
+
+   Structure containing the fields for the datetime C API.
+
+   The fields of this structure are private and subject to change.
+
+   Do not use this directly; prefer ``PyDateTime_*`` APIs instead.
+
+.. c:var:: PyDateTime_CAPI *PyDateTimeAPI
+
+   Dynamically allocated object containing the datetime C API.
+
+   This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds.
+
 .. c:type:: PyDateTime_Date
 
    This subtype of :c:type:`PyObject` represents a Python date object.
@@ -46,7 +77,7 @@ macros.
 
 .. c:var:: PyTypeObject PyDateTime_DeltaType
 
-   This instance of :c:type:`PyTypeObject` represents Python type for
+   This instance of :c:type:`PyTypeObject` represents the Python type for
    the difference between two datetime values;
    it is the same object as :class:`datetime.timedelta` in the Python layer.
 
@@ -325,3 +356,16 @@ Macros for the convenience of modules implementing the DB API:
 
    Create and return a new :class:`datetime.date` object given an argument
    tuple suitable for passing to :meth:`datetime.date.fromtimestamp`.
+
+
+Internal data
+-------------
+
+The following symbols are exposed by the C API but should be considered
+internal-only.
+
+.. c:macro:: PyDateTime_CAPSULE_NAME
+
+   Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`.
+
+   Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead.

Doc/c-api/descriptor.rst

@@ -21,20 +21,104 @@ found in the dictionary of type objects.
 .. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth)
 
 
+.. c:var:: PyTypeObject PyMemberDescr_Type
+
+   The type object for member descriptor objects created from
+   :c:type:`PyMemberDef` structures. These descriptors expose fields of a
+   C struct as attributes on a type, and correspond
+   to :class:`types.MemberDescriptorType` objects in Python.
+
+
+
+.. c:var:: PyTypeObject PyGetSetDescr_Type
+
+   The type object for get/set descriptor objects created from
+   :c:type:`PyGetSetDef` structures. These descriptors implement attributes
+   whose value is computed by C getter and setter functions, and are used
+   for many built-in type attributes.
+
+
 .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth)
 
 
+.. c:var:: PyTypeObject PyMethodDescr_Type
+
+   The type object for method descriptor objects created from
+   :c:type:`PyMethodDef` structures. These descriptors expose C functions as
+   methods on a type, and correspond to :class:`types.MemberDescriptorType`
+   objects in Python.
+
+
 .. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped)
 
 
+.. c:var:: PyTypeObject PyWrapperDescr_Type
+
+   The type object for wrapper descriptor objects created by
+   :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper
+   descriptors are used internally to expose special methods implemented
+   via wrapper structures, and appear in Python as
+   :class:`types.WrapperDescriptorType` objects.
+
+
 .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)
 
 
 .. c:function:: int PyDescr_IsData(PyObject *descr)
 
-   Return non-zero if the descriptor objects *descr* describes a data attribute, or
+   Return non-zero if the descriptor object *descr* describes a data attribute, or
    ``0`` if it describes a method.  *descr* must be a descriptor object; there is
    no error checking.
 
 
 .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *)
+
+
+Built-in descriptors
+^^^^^^^^^^^^^^^^^^^^
+
+.. c:var:: PyTypeObject PySuper_Type
+
+   The type object for super objects. This is the same object as
+   :class:`super` in the Python layer.
+
+
+.. c:var:: PyTypeObject PyClassMethod_Type
+
+   The type of class method objects. This is the same object as
+   :class:`classmethod` in the Python layer.
+
+
+.. c:var:: PyTypeObject PyClassMethodDescr_Type
+
+   The type object for C-level class method descriptor objects.
+   This is the type of the descriptors created for :func:`classmethod` defined in
+   C extension types, and is the same object as :class:`classmethod`
+   in Python.
+
+
+.. c:function:: PyObject *PyClassMethod_New(PyObject *callable)
+
+   Create a new :class:`classmethod` object wrapping *callable*.
+   *callable* must be a callable object and must not be ``NULL``.
+
+   On success, this function returns a :term:`strong reference` to a new class
+   method descriptor. On failure, this function returns ``NULL`` with an
+   exception set.
+
+
+.. c:var:: PyTypeObject PyStaticMethod_Type
+
+   The type of static method objects. This is the same object as
+   :class:`staticmethod` in the Python layer.
+
+
+.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable)
+
+   Create a new :class:`staticmethod` object wrapping *callable*.
+   *callable* must be a callable object and must not be ``NULL``.
+
+   On success, this function returns a :term:`strong reference` to a new static
+   method descriptor. On failure, this function returns ``NULL`` with an
+   exception set.
+

Doc/c-api/dict.rst

@@ -43,6 +43,17 @@ Dictionary Objects
    prevent modification of the dictionary for non-dynamic class types.
 
 
+.. c:var:: PyTypeObject PyDictProxy_Type
+
+   The type object for mapping proxy objects created by
+   :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute
+   of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a
+   dynamic, read-only view of an underlying dictionary: changes to the
+   underlying dictionary are reflected in the proxy, but the proxy itself
+   does not support mutation operations. This corresponds to
+   :class:`types.MappingProxyType` in Python.
+
+
 .. c:function:: void PyDict_Clear(PyObject *p)
 
    Empty an existing dictionary of all key-value pairs.
@@ -50,7 +61,7 @@ Dictionary Objects
 
 .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key)
 
-   Determine if dictionary *p* contains *key*.  If an item in *p* is matches
+   Determine if dictionary *p* contains *key*.  If an item in *p* matches
    *key*, return ``1``, otherwise return ``0``.  On error, return ``-1``.
    This is equivalent to the Python expression ``key in p``.
 
@@ -198,7 +209,7 @@ Dictionary Objects
 .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result)
 
    Remove *key* from dictionary *p* and optionally return the removed value.
-   Do not raise :exc:`KeyError` if the key missing.
+   Do not raise :exc:`KeyError` if the key is missing.
 
    - If the key is present, set *\*result* to a new reference to the removed
      value if *result* is not ``NULL``, and return ``1``.
@@ -207,7 +218,7 @@ Dictionary Objects
    - On error, raise an exception and return ``-1``.
 
    Similar to :meth:`dict.pop`, but without the default value and
-   not raising :exc:`KeyError` if the key missing.
+   not raising :exc:`KeyError` if the key is missing.
 
    .. versionadded:: 3.13
 
@@ -245,6 +256,11 @@ Dictionary Objects
    ``len(p)`` on a dictionary.
 
 
+.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p)
+
+   Similar to :c:func:`PyDict_Size`, but without error checking.
+
+
 .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue)
 
    Iterate over all key-value pairs in the dictionary *p*.  The
@@ -301,6 +317,15 @@ Dictionary Objects
       }
       Py_END_CRITICAL_SECTION();
 
+   .. note::
+
+      On the free-threaded build, this function can be used safely inside a
+      critical section. However, the references returned for *pkey* and *pvalue*
+      are :term:`borrowed <borrowed reference>` and are only valid while the
+      critical section is held. If you need to use these objects outside the
+      critical section or when the critical section can be suspended, create a
+      :term:`strong reference <strong reference>` (for example, using
+      :c:func:`Py_NewRef`).
 
 .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override)
 
@@ -417,3 +442,138 @@ Dictionary Objects
    it before returning.
 
    .. versionadded:: 3.12
+
+
+Dictionary View Objects
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. c:function:: int PyDictViewSet_Check(PyObject *op)
+
+   Return true if *op* is a view of a set inside a dictionary. This is currently
+   equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This
+   function always succeeds.
+
+
+.. c:var:: PyTypeObject PyDictKeys_Type
+
+   Type object for a view of dictionary keys. In Python, this is the type of
+   the object returned by :meth:`dict.keys`.
+
+
+.. c:function:: int PyDictKeys_Check(PyObject *op)
+
+   Return true if *op* is an instance of a dictionary keys view. This function
+   always succeeds.
+
+
+.. c:var:: PyTypeObject PyDictValues_Type
+
+   Type object for a view of dictionary values. In Python, this is the type of
+   the object returned by :meth:`dict.values`.
+
+
+.. c:function:: int PyDictValues_Check(PyObject *op)
+
+   Return true if *op* is an instance of a dictionary values view. This function
+   always succeeds.
+
+
+.. c:var:: PyTypeObject PyDictItems_Type
+
+   Type object for a view of dictionary items. In Python, this is the type of
+   the object returned by :meth:`dict.items`.
+
+
+.. c:function:: int PyDictItems_Check(PyObject *op)
+
+   Return true if *op* is an instance of a dictionary items view. This function
+   always succeeds.
+
+
+Ordered Dictionaries
+^^^^^^^^^^^^^^^^^^^^
+
+Python's C API provides interface for :class:`collections.OrderedDict` from C.
+Since Python 3.7, dictionaries are ordered by default, so there is usually
+little need for these functions; prefer ``PyDict*`` where possible.
+
+
+.. c:var:: PyTypeObject PyODict_Type
+
+   Type object for ordered dictionaries. This is the same object as
+   :class:`collections.OrderedDict` in the Python layer.
+
+
+.. c:function:: int PyODict_Check(PyObject *od)
+
+   Return true if *od* is an ordered dictionary object or an instance of a
+   subtype of the :class:`~collections.OrderedDict` type.  This function
+   always succeeds.
+
+
+.. c:function:: int PyODict_CheckExact(PyObject *od)
+
+   Return true if *od* is an ordered dictionary object, but not an instance of
+   a subtype of the :class:`~collections.OrderedDict` type.
+   This function always succeeds.
+
+
+.. c:var:: PyTypeObject PyODictKeys_Type
+
+   Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries.
+
+
+.. c:var:: PyTypeObject PyODictValues_Type
+
+   Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries.
+
+
+.. c:var:: PyTypeObject PyODictItems_Type
+
+   Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries.
+
+
+.. c:function:: PyObject *PyODict_New(void)
+
+   Return a new empty ordered dictionary, or ``NULL`` on failure.
+
+   This is analogous to :c:func:`PyDict_New`.
+
+
+.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value)
+
+   Insert *value* into the ordered dictionary *od* with a key of *key*.
+   Return ``0`` on success or ``-1`` with an exception set on failure.
+
+   This is analogous to :c:func:`PyDict_SetItem`.
+
+
+.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key)
+
+   Remove the entry in the ordered dictionary *od* with key *key*.
+   Return ``0`` on success or ``-1`` with an exception set on failure.
+
+   This is analogous to :c:func:`PyDict_DelItem`.
+
+
+These are :term:`soft deprecated` aliases to ``PyDict`` APIs:
+
+
+.. list-table::
+   :widths: auto
+   :header-rows: 1
+
+   * * ``PyODict``
+     * ``PyDict``
+   * * .. c:macro:: PyODict_GetItem(od, key)
+     * :c:func:`PyDict_GetItem`
+   * * .. c:macro:: PyODict_GetItemWithError(od, key)
+     * :c:func:`PyDict_GetItemWithError`
+   * * .. c:macro:: PyODict_GetItemString(od, key)
+     * :c:func:`PyDict_GetItemString`
+   * * .. c:macro:: PyODict_Contains(od, key)
+     * :c:func:`PyDict_Contains`
+   * * .. c:macro:: PyODict_Size(od)
+     * :c:func:`PyDict_Size`
+   * * .. c:macro:: PyODict_SIZE(od)
+     * :c:func:`PyDict_GET_SIZE`

Doc/c-api/exceptions.rst

@@ -309,6 +309,14 @@ For convenience, some of these functions will always return a
    .. versionadded:: 3.4
 
 
+.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset)
+
+   Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the
+   *end_lineno* and *end_col_offset* information for the current exception.
+
+   .. versionadded:: 3.10
+
+
 .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset)
 
    Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string
@@ -331,6 +339,23 @@ For convenience, some of these functions will always return a
    use.
 
 
+.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno)
+
+   Get the source line in *filename* at line *lineno*. *filename* should be a
+   Python :class:`str` object.
+
+   On success, this function returns a Python string object with the found line.
+   On failure, this function returns ``NULL`` without an exception set.
+
+
+.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno)
+
+   Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a
+   :c:expr:`const char *`, which is decoded with the
+   :term:`filesystem encoding and error handler`, instead of a
+   Python object reference.
+
+
 Issuing warnings
 ================
 
@@ -394,6 +419,15 @@ an error value).
    .. versionadded:: 3.2
 
 
+.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...)
+
+   Similar to :c:func:`PyErr_WarnExplicit`, but uses
+   :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is
+   an ASCII-encoded string.
+
+   .. versionadded:: 3.2
+
+
 .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...)
 
    Function similar to :c:func:`PyErr_WarnFormat`, but *category* is
@@ -749,9 +783,30 @@ Exception Classes
    .. versionadded:: 3.2
 
 
+.. c:function:: int PyExceptionClass_Check(PyObject *ob)
+
+   Return non-zero if *ob* is an exception class, zero otherwise. This function always succeeds.
+
+
+.. c:function:: const char *PyExceptionClass_Name(PyObject *ob)
+
+   Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*.
+
+
 Exception Objects
 =================
 
+.. c:function:: int PyExceptionInstance_Check(PyObject *op)
+
+   Return true if *op* is an instance of :class:`BaseException`, false
+   otherwise. This function always succeeds.
+
+
+.. c:macro:: PyExceptionInstance_Class(op)
+
+   Equivalent to :c:func:`Py_TYPE(op) <Py_TYPE>`.
+
+
 .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex)
 
    Return the traceback associated with the exception as a new reference, as
@@ -929,6 +984,9 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
    be concatenated to the :exc:`RecursionError` message caused by the recursion
    depth limit.
 
+   .. seealso::
+      The :c:func:`PyUnstable_ThreadState_SetStackProtection` function.
+
    .. versionchanged:: 3.9
       This function is now also available in the :ref:`limited API <limited-c-api>`.
 
@@ -969,184 +1027,159 @@ these are the C equivalent to :func:`reprlib.recursive_repr`.
    Ends a :c:func:`Py_ReprEnter`.  Must be called once for each
    invocation of :c:func:`Py_ReprEnter` that returns zero.
 
+.. c:function:: int Py_GetRecursionLimit(void)
+
+   Get the recursion limit for the current interpreter. It can be set with
+   :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the
+   Python interpreter stack from growing infinitely.
+
+   This function cannot fail, and the caller must hold an
+   :term:`attached thread state`.
+
+   .. seealso::
+      :py:func:`sys.getrecursionlimit`
+
+.. c:function:: void Py_SetRecursionLimit(int new_limit)
+
+   Set the recursion limit for the current interpreter.
+
+   This function cannot fail, and the caller must hold an
+   :term:`attached thread state`.
+
+   .. seealso::
+      :py:func:`sys.setrecursionlimit`
 
 .. _standardexceptions:
 
-Standard Exceptions
-===================
+Exception and warning types
+===========================
 
-All standard Python exceptions are available as global variables whose names are
-``PyExc_`` followed by the Python exception name.  These have the type
-:c:expr:`PyObject*`; they are all class objects.  For completeness, here are all
-the variables:
+All standard Python exceptions and warning categories are available as global
+variables whose names are ``PyExc_`` followed by the Python exception name.
+These have the type :c:expr:`PyObject*`; they are all class objects.
 
-.. index::
-   single: PyExc_BaseException (C var)
-   single: PyExc_Exception (C var)
-   single: PyExc_ArithmeticError (C var)
-   single: PyExc_AssertionError (C var)
-   single: PyExc_AttributeError (C var)
-   single: PyExc_BlockingIOError (C var)
-   single: PyExc_BrokenPipeError (C var)
-   single: PyExc_BufferError (C var)
-   single: PyExc_ChildProcessError (C var)
-   single: PyExc_ConnectionAbortedError (C var)
-   single: PyExc_ConnectionError (C var)
-   single: PyExc_ConnectionRefusedError (C var)
-   single: PyExc_ConnectionResetError (C var)
-   single: PyExc_EOFError (C var)
-   single: PyExc_FileExistsError (C var)
-   single: PyExc_FileNotFoundError (C var)
-   single: PyExc_FloatingPointError (C var)
-   single: PyExc_GeneratorExit (C var)
-   single: PyExc_ImportError (C var)
-   single: PyExc_IndentationError (C var)
-   single: PyExc_IndexError (C var)
-   single: PyExc_InterruptedError (C var)
-   single: PyExc_IsADirectoryError (C var)
-   single: PyExc_KeyError (C var)
-   single: PyExc_KeyboardInterrupt (C var)
-   single: PyExc_LookupError (C var)
-   single: PyExc_MemoryError (C var)
-   single: PyExc_ModuleNotFoundError (C var)
-   single: PyExc_NameError (C var)
-   single: PyExc_NotADirectoryError (C var)
-   single: PyExc_NotImplementedError (C var)
-   single: PyExc_OSError (C var)
-   single: PyExc_OverflowError (C var)
-   single: PyExc_PermissionError (C var)
-   single: PyExc_ProcessLookupError (C var)
-   single: PyExc_PythonFinalizationError (C var)
-   single: PyExc_RecursionError (C var)
-   single: PyExc_ReferenceError (C var)
-   single: PyExc_RuntimeError (C var)
-   single: PyExc_StopAsyncIteration (C var)
-   single: PyExc_StopIteration (C var)
-   single: PyExc_SyntaxError (C var)
-   single: PyExc_SystemError (C var)
-   single: PyExc_SystemExit (C var)
-   single: PyExc_TabError (C var)
-   single: PyExc_TimeoutError (C var)
-   single: PyExc_TypeError (C var)
-   single: PyExc_UnboundLocalError (C var)
-   single: PyExc_UnicodeDecodeError (C var)
-   single: PyExc_UnicodeEncodeError (C var)
-   single: PyExc_UnicodeError (C var)
-   single: PyExc_UnicodeTranslateError (C var)
-   single: PyExc_ValueError (C var)
-   single: PyExc_ZeroDivisionError (C var)
+For completeness, here are all the variables:
 
-+-----------------------------------------+---------------------------------+----------+
-| C Name                                  | Python Name                     | Notes    |
-+=========================================+=================================+==========+
-| :c:data:`PyExc_BaseException`           | :exc:`BaseException`            | [1]_     |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_Exception`               | :exc:`Exception`                | [1]_     |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ArithmeticError`         | :exc:`ArithmeticError`          | [1]_     |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_AssertionError`          | :exc:`AssertionError`           |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_AttributeError`          | :exc:`AttributeError`           |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_BlockingIOError`         | :exc:`BlockingIOError`          |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_BrokenPipeError`         | :exc:`BrokenPipeError`          |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_BufferError`             | :exc:`BufferError`              |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ChildProcessError`       | :exc:`ChildProcessError`        |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ConnectionAbortedError`  | :exc:`ConnectionAbortedError`   |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ConnectionError`         | :exc:`ConnectionError`          |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ConnectionRefusedError`  | :exc:`ConnectionRefusedError`   |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ConnectionResetError`    | :exc:`ConnectionResetError`     |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_EOFError`                | :exc:`EOFError`                 |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_FileExistsError`         | :exc:`FileExistsError`          |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_FileNotFoundError`       | :exc:`FileNotFoundError`        |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_FloatingPointError`      | :exc:`FloatingPointError`       |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_GeneratorExit`           | :exc:`GeneratorExit`            |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ImportError`             | :exc:`ImportError`              |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_IndentationError`        | :exc:`IndentationError`         |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_IndexError`              | :exc:`IndexError`               |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_InterruptedError`        | :exc:`InterruptedError`         |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_IsADirectoryError`       | :exc:`IsADirectoryError`        |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_KeyError`                | :exc:`KeyError`                 |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_KeyboardInterrupt`       | :exc:`KeyboardInterrupt`        |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_LookupError`             | :exc:`LookupError`              | [1]_     |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_MemoryError`             | :exc:`MemoryError`              |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ModuleNotFoundError`     | :exc:`ModuleNotFoundError`      |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_NameError`               | :exc:`NameError`                |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_NotADirectoryError`      | :exc:`NotADirectoryError`       |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_NotImplementedError`     | :exc:`NotImplementedError`      |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_OSError`                 | :exc:`OSError`                  | [1]_     |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_OverflowError`           | :exc:`OverflowError`            |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_PermissionError`         | :exc:`PermissionError`          |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ProcessLookupError`      | :exc:`ProcessLookupError`       |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError`  |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_RecursionError`          | :exc:`RecursionError`           |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ReferenceError`          | :exc:`ReferenceError`           |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_RuntimeError`            | :exc:`RuntimeError`             |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_StopAsyncIteration`      | :exc:`StopAsyncIteration`       |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_StopIteration`           | :exc:`StopIteration`            |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_SyntaxError`             | :exc:`SyntaxError`              |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_SystemError`             | :exc:`SystemError`              |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_SystemExit`              | :exc:`SystemExit`               |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_TabError`                | :exc:`TabError`                 |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_TimeoutError`            | :exc:`TimeoutError`             |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_TypeError`               | :exc:`TypeError`                |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UnboundLocalError`       | :exc:`UnboundLocalError`        |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UnicodeDecodeError`      | :exc:`UnicodeDecodeError`       |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UnicodeEncodeError`      | :exc:`UnicodeEncodeError`       |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UnicodeError`            | :exc:`UnicodeError`             |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UnicodeTranslateError`   | :exc:`UnicodeTranslateError`    |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ValueError`              | :exc:`ValueError`               |          |
-+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ZeroDivisionError`       | :exc:`ZeroDivisionError`        |          |
-+-----------------------------------------+---------------------------------+----------+
+Exception types
+---------------
+
+.. list-table::
+   :align: left
+   :widths: auto
+   :header-rows: 1
+
+   * * C name
+     * Python name
+   * * .. c:var:: PyObject *PyExc_BaseException
+     * :exc:`BaseException`
+   * * .. c:var:: PyObject *PyExc_BaseExceptionGroup
+     * :exc:`BaseExceptionGroup`
+   * * .. c:var:: PyObject *PyExc_Exception
+     * :exc:`Exception`
+   * * .. c:var:: PyObject *PyExc_ArithmeticError
+     * :exc:`ArithmeticError`
+   * * .. c:var:: PyObject *PyExc_AssertionError
+     * :exc:`AssertionError`
+   * * .. c:var:: PyObject *PyExc_AttributeError
+     * :exc:`AttributeError`
+   * * .. c:var:: PyObject *PyExc_BlockingIOError
+     * :exc:`BlockingIOError`
+   * * .. c:var:: PyObject *PyExc_BrokenPipeError
+     * :exc:`BrokenPipeError`
+   * * .. c:var:: PyObject *PyExc_BufferError
+     * :exc:`BufferError`
+   * * .. c:var:: PyObject *PyExc_ChildProcessError
+     * :exc:`ChildProcessError`
+   * * .. c:var:: PyObject *PyExc_ConnectionAbortedError
+     * :exc:`ConnectionAbortedError`
+   * * .. c:var:: PyObject *PyExc_ConnectionError
+     * :exc:`ConnectionError`
+   * * .. c:var:: PyObject *PyExc_ConnectionRefusedError
+     * :exc:`ConnectionRefusedError`
+   * * .. c:var:: PyObject *PyExc_ConnectionResetError
+     * :exc:`ConnectionResetError`
+   * * .. c:var:: PyObject *PyExc_EOFError
+     * :exc:`EOFError`
+   * * .. c:var:: PyObject *PyExc_FileExistsError
+     * :exc:`FileExistsError`
+   * * .. c:var:: PyObject *PyExc_FileNotFoundError
+     * :exc:`FileNotFoundError`
+   * * .. c:var:: PyObject *PyExc_FloatingPointError
+     * :exc:`FloatingPointError`
+   * * .. c:var:: PyObject *PyExc_GeneratorExit
+     * :exc:`GeneratorExit`
+   * * .. c:var:: PyObject *PyExc_ImportError
+     * :exc:`ImportError`
+   * * .. c:var:: PyObject *PyExc_IndentationError
+     * :exc:`IndentationError`
+   * * .. c:var:: PyObject *PyExc_IndexError
+     * :exc:`IndexError`
+   * * .. c:var:: PyObject *PyExc_InterruptedError
+     * :exc:`InterruptedError`
+   * * .. c:var:: PyObject *PyExc_IsADirectoryError
+     * :exc:`IsADirectoryError`
+   * * .. c:var:: PyObject *PyExc_KeyError
+     * :exc:`KeyError`
+   * * .. c:var:: PyObject *PyExc_KeyboardInterrupt
+     * :exc:`KeyboardInterrupt`
+   * * .. c:var:: PyObject *PyExc_LookupError
+     * :exc:`LookupError`
+   * * .. c:var:: PyObject *PyExc_MemoryError
+     * :exc:`MemoryError`
+   * * .. c:var:: PyObject *PyExc_ModuleNotFoundError
+     * :exc:`ModuleNotFoundError`
+   * * .. c:var:: PyObject *PyExc_NameError
+     * :exc:`NameError`
+   * * .. c:var:: PyObject *PyExc_NotADirectoryError
+     * :exc:`NotADirectoryError`
+   * * .. c:var:: PyObject *PyExc_NotImplementedError
+     * :exc:`NotImplementedError`
+   * * .. c:var:: PyObject *PyExc_OSError
+     * :exc:`OSError`
+   * * .. c:var:: PyObject *PyExc_OverflowError
+     * :exc:`OverflowError`
+   * * .. c:var:: PyObject *PyExc_PermissionError
+     * :exc:`PermissionError`
+   * * .. c:var:: PyObject *PyExc_ProcessLookupError
+     * :exc:`ProcessLookupError`
+   * * .. c:var:: PyObject *PyExc_PythonFinalizationError
+     * :exc:`PythonFinalizationError`
+   * * .. c:var:: PyObject *PyExc_RecursionError
+     * :exc:`RecursionError`
+   * * .. c:var:: PyObject *PyExc_ReferenceError
+     * :exc:`ReferenceError`
+   * * .. c:var:: PyObject *PyExc_RuntimeError
+     * :exc:`RuntimeError`
+   * * .. c:var:: PyObject *PyExc_StopAsyncIteration
+     * :exc:`StopAsyncIteration`
+   * * .. c:var:: PyObject *PyExc_StopIteration
+     * :exc:`StopIteration`
+   * * .. c:var:: PyObject *PyExc_SyntaxError
+     * :exc:`SyntaxError`
+   * * .. c:var:: PyObject *PyExc_SystemError
+     * :exc:`SystemError`
+   * * .. c:var:: PyObject *PyExc_SystemExit
+     * :exc:`SystemExit`
+   * * .. c:var:: PyObject *PyExc_TabError
+     * :exc:`TabError`
+   * * .. c:var:: PyObject *PyExc_TimeoutError
+     * :exc:`TimeoutError`
+   * * .. c:var:: PyObject *PyExc_TypeError
+     * :exc:`TypeError`
+   * * .. c:var:: PyObject *PyExc_UnboundLocalError
+     * :exc:`UnboundLocalError`
+   * * .. c:var:: PyObject *PyExc_UnicodeDecodeError
+     * :exc:`UnicodeDecodeError`
+   * * .. c:var:: PyObject *PyExc_UnicodeEncodeError
+     * :exc:`UnicodeEncodeError`
+   * * .. c:var:: PyObject *PyExc_UnicodeError
+     * :exc:`UnicodeError`
+   * * .. c:var:: PyObject *PyExc_UnicodeTranslateError
+     * :exc:`UnicodeTranslateError`
+   * * .. c:var:: PyObject *PyExc_ValueError
+     * :exc:`ValueError`
+   * * .. c:var:: PyObject *PyExc_ZeroDivisionError
+     * :exc:`ZeroDivisionError`
 
 .. versionadded:: 3.3
    :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`,
@@ -1164,88 +1197,116 @@ the variables:
 .. versionadded:: 3.6
    :c:data:`PyExc_ModuleNotFoundError`.
 
-These are compatibility aliases to :c:data:`PyExc_OSError`:
+.. versionadded:: 3.11
+   :c:data:`PyExc_BaseExceptionGroup`.
 
-.. index::
-   single: PyExc_EnvironmentError (C var)
-   single: PyExc_IOError (C var)
-   single: PyExc_WindowsError (C var)
 
-+-------------------------------------+----------+
-| C Name                              | Notes    |
-+=====================================+==========+
-| :c:data:`!PyExc_EnvironmentError`   |          |
-+-------------------------------------+----------+
-| :c:data:`!PyExc_IOError`            |          |
-+-------------------------------------+----------+
-| :c:data:`!PyExc_WindowsError`       | [2]_     |
-+-------------------------------------+----------+
+OSError aliases
+---------------
+
+The following are a compatibility aliases to :c:data:`PyExc_OSError`.
 
 .. versionchanged:: 3.3
    These aliases used to be separate exception types.
 
+.. list-table::
+   :align: left
+   :widths: auto
+   :header-rows: 1
+
+   * * C name
+     * Python name
+     * Notes
+   * * .. c:var:: PyObject *PyExc_EnvironmentError
+     * :exc:`OSError`
+     *
+   * * .. c:var:: PyObject *PyExc_IOError
+     * :exc:`OSError`
+     *
+   * * .. c:var:: PyObject *PyExc_WindowsError
+     * :exc:`OSError`
+     * [win]_
+
 Notes:
 
-.. [1]
-   This is a base class for other standard exceptions.
+.. [win]
+   :c:var:`!PyExc_WindowsError` is only defined on Windows; protect code that
+   uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined.
 
-.. [2]
-   Only defined on Windows; protect code that uses this by testing that the
-   preprocessor macro ``MS_WINDOWS`` is defined.
 
 .. _standardwarningcategories:
 
-Standard Warning Categories
-===========================
+Warning types
+-------------
 
-All standard Python warning categories are available as global variables whose
-names are ``PyExc_`` followed by the Python exception name. These have the type
-:c:expr:`PyObject*`; they are all class objects. For completeness, here are all
-the variables:
+.. list-table::
+   :align: left
+   :widths: auto
+   :header-rows: 1
 
-.. index::
-   single: PyExc_Warning (C var)
-   single: PyExc_BytesWarning (C var)
-   single: PyExc_DeprecationWarning (C var)
-   single: PyExc_FutureWarning (C var)
-   single: PyExc_ImportWarning (C var)
-   single: PyExc_PendingDeprecationWarning (C var)
-   single: PyExc_ResourceWarning (C var)
-   single: PyExc_RuntimeWarning (C var)
-   single: PyExc_SyntaxWarning (C var)
-   single: PyExc_UnicodeWarning (C var)
-   single: PyExc_UserWarning (C var)
-
-+------------------------------------------+---------------------------------+----------+
-| C Name                                   | Python Name                     | Notes    |
-+==========================================+=================================+==========+
-| :c:data:`PyExc_Warning`                  | :exc:`Warning`                  | [3]_     |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_BytesWarning`             | :exc:`BytesWarning`             |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_DeprecationWarning`       | :exc:`DeprecationWarning`       |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_FutureWarning`            | :exc:`FutureWarning`            |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ImportWarning`            | :exc:`ImportWarning`            |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`|          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ResourceWarning`          | :exc:`ResourceWarning`          |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_RuntimeWarning`           | :exc:`RuntimeWarning`           |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_SyntaxWarning`            | :exc:`SyntaxWarning`            |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UnicodeWarning`           | :exc:`UnicodeWarning`           |          |
-+------------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_UserWarning`              | :exc:`UserWarning`              |          |
-+------------------------------------------+---------------------------------+----------+
+   * * C name
+     * Python name
+   * * .. c:var:: PyObject *PyExc_Warning
+     * :exc:`Warning`
+   * * .. c:var:: PyObject *PyExc_BytesWarning
+     * :exc:`BytesWarning`
+   * * .. c:var:: PyObject *PyExc_DeprecationWarning
+     * :exc:`DeprecationWarning`
+   * * .. c:var:: PyObject *PyExc_EncodingWarning
+     * :exc:`EncodingWarning`
+   * * .. c:var:: PyObject *PyExc_FutureWarning
+     * :exc:`FutureWarning`
+   * * .. c:var:: PyObject *PyExc_ImportWarning
+     * :exc:`ImportWarning`
+   * * .. c:var:: PyObject *PyExc_PendingDeprecationWarning
+     * :exc:`PendingDeprecationWarning`
+   * * .. c:var:: PyObject *PyExc_ResourceWarning
+     * :exc:`ResourceWarning`
+   * * .. c:var:: PyObject *PyExc_RuntimeWarning
+     * :exc:`RuntimeWarning`
+   * * .. c:var:: PyObject *PyExc_SyntaxWarning
+     * :exc:`SyntaxWarning`
+   * * .. c:var:: PyObject *PyExc_UnicodeWarning
+     * :exc:`UnicodeWarning`
+   * * .. c:var:: PyObject *PyExc_UserWarning
+     * :exc:`UserWarning`
 
 .. versionadded:: 3.2
    :c:data:`PyExc_ResourceWarning`.
 
-Notes:
+.. versionadded:: 3.10
+   :c:data:`PyExc_EncodingWarning`.
 
-.. [3]
-   This is a base class for other standard warning categories.
+
+Tracebacks
+==========
+
+.. c:var:: PyTypeObject PyTraceBack_Type
+
+   Type object for traceback objects. This is available as
+   :class:`types.TracebackType` in the Python layer.
+
+
+.. c:function:: int PyTraceBack_Check(PyObject *op)
+
+   Return true if *op* is a traceback object, false otherwise. This function
+   does not account for subtypes.
+
+
+.. c:function:: int PyTraceBack_Here(PyFrameObject *f)
+
+   Replace the :attr:`~BaseException.__traceback__` attribute on the current
+   exception with a new traceback prepending *f* to the existing chain.
+
+   Calling this function without an exception set is undefined behavior.
+
+   This function returns ``0`` on success, and returns ``-1`` with an
+   exception set on failure.
+
+
+.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f)
+
+   Write the traceback *tb* into the file *f*.
+
+   This function returns ``0`` on success, and returns ``-1`` with an
+   exception set on failure.

Doc/c-api/extension-modules.rst

@@ -0,0 +1,247 @@
+.. highlight:: c
+
+.. _extension-modules:
+
+Defining extension modules
+--------------------------
+
+A C extension for CPython is a shared library (for example, a ``.so`` file
+on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process
+(for example, it is compiled with compatible compiler settings), and which
+exports an :ref:`initialization function <extension-export-hook>`.
+
+To be importable by default (that is, by
+:py:class:`importlib.machinery.ExtensionFileLoader`),
+the shared library must be available on :py:attr:`sys.path`,
+and must be named after the module name plus an extension listed in
+:py:attr:`importlib.machinery.EXTENSION_SUFFIXES`.
+
+.. note::
+
+   Building, packaging and distributing extension modules is best done with
+   third-party tools, and is out of scope of this document.
+   One suitable tool is Setuptools, whose documentation can be found at
+   https://setuptools.pypa.io/en/latest/setuptools.html.
+
+Normally, the initialization function returns a module definition initialized
+using :c:func:`PyModuleDef_Init`.
+This allows splitting the creation process into several phases:
+
+- Before any substantial code is executed, Python can determine which
+  capabilities the module supports, and it can adjust the environment or
+  refuse loading an incompatible extension.
+- By default, Python itself creates the module object -- that is, it does
+  the equivalent of :py:meth:`object.__new__` for classes.
+  It also sets initial attributes like :attr:`~module.__package__` and
+  :attr:`~module.__loader__`.
+- Afterwards, the module object is initialized using extension-specific
+  code -- the equivalent of :py:meth:`~object.__init__` on classes.
+
+This is called *multi-phase initialization* to distinguish it from the legacy
+(but still supported) *single-phase initialization* scheme,
+where the initialization function returns a fully constructed module.
+See the :ref:`single-phase-initialization section below <single-phase-initialization>`
+for details.
+
+.. versionchanged:: 3.5
+
+   Added support for multi-phase initialization (:pep:`489`).
+
+
+Multiple module instances
+.........................
+
+By default, extension modules are not singletons.
+For example, if the :py:attr:`sys.modules` entry is removed and the module
+is re-imported, a new module object is created, and typically populated with
+fresh method and type objects.
+The old module is subject to normal garbage collection.
+This mirrors the behavior of pure-Python modules.
+
+Additional module instances may be created in
+:ref:`sub-interpreters <sub-interpreter-support>`
+or after Python runtime reinitialization
+(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`).
+In these cases, sharing Python objects between module instances would likely
+cause crashes or undefined behavior.
+
+To avoid such issues, each instance of an extension module should
+be *isolated*: changes to one instance should not implicitly affect the others,
+and all state owned by the module, including references to Python objects,
+should be specific to a particular module instance.
+See :ref:`isolating-extensions-howto` for more details and a practical guide.
+
+A simpler way to avoid these issues is
+:ref:`raising an error on repeated initialization <isolating-extensions-optout>`.
+
+All modules are expected to support
+:ref:`sub-interpreters <sub-interpreter-support>`, or otherwise explicitly
+signal a lack of support.
+This is usually achieved by isolation or blocking repeated initialization,
+as above.
+A module may also be limited to the main interpreter using
+the :c:data:`Py_mod_multiple_interpreters` slot.
+
+
+.. _extension-export-hook:
+
+Initialization function
+.......................
+
+The initialization function defined by an extension module has the
+following signature:
+
+.. c:function:: PyObject* PyInit_modulename(void)
+
+Its name should be :samp:`PyInit_{<name>}`, with ``<name>`` replaced by the
+name of the module.
+
+For modules with ASCII-only names, the function must instead be named
+:samp:`PyInit_{<name>}`, with ``<name>`` replaced by the name of the module.
+When using :ref:`multi-phase-initialization`, non-ASCII module names
+are allowed. In this case, the initialization function name is
+:samp:`PyInitU_{<name>}`, with ``<name>`` encoded using Python's
+*punycode* encoding with hyphens replaced by underscores. In Python:
+
+.. code-block:: python
+
+    def initfunc_name(name):
+        try:
+            suffix = b'_' + name.encode('ascii')
+        except UnicodeEncodeError:
+            suffix = b'U_' + name.encode('punycode').replace(b'-', b'_')
+        return b'PyInit' + suffix
+
+It is recommended to define the initialization function using a helper macro:
+
+.. c:macro:: PyMODINIT_FUNC
+
+   Declare an extension module initialization function.
+   This macro:
+
+   * specifies the :c:expr:`PyObject*` return type,
+   * adds any special linkage declarations required by the platform, and
+   * for C++, declares the function as ``extern "C"``.
+
+For example, a module called ``spam`` would be defined like this::
+
+   static struct PyModuleDef spam_module = {
+       .m_base = PyModuleDef_HEAD_INIT,
+       .m_name = "spam",
+       ...
+   };
+
+   PyMODINIT_FUNC
+   PyInit_spam(void)
+   {
+       return PyModuleDef_Init(&spam_module);
+   }
+
+It is possible to export multiple modules from a single shared library by
+defining multiple initialization functions. However, importing them requires
+using symbolic links or a custom importer, because by default only the
+function corresponding to the filename is found.
+See the `Multiple modules in one library <https://peps.python.org/pep-0489/#multiple-modules-in-one-library>`__
+section in :pep:`489` for details.
+
+The initialization function is typically the only non-\ ``static``
+item defined in the module's C source.
+
+
+.. _multi-phase-initialization:
+
+Multi-phase initialization
+..........................
+
+Normally, the :ref:`initialization function <extension-export-hook>`
+(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with
+non-``NULL`` :c:member:`~PyModuleDef.m_slots`.
+Before it is returned, the ``PyModuleDef`` instance must be initialized
+using the following function:
+
+
+.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def)
+
+   Ensure a module definition is a properly initialized Python object that
+   correctly reports its type and a reference count.
+
+   Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred.
+
+   Calling this function is required for :ref:`multi-phase-initialization`.
+   It should not be used in other contexts.
+
+   Note that Python assumes that ``PyModuleDef`` structures are statically
+   allocated.
+   This function may return either a new reference or a borrowed one;
+   this reference must not be released.
+
+   .. versionadded:: 3.5
+
+
+.. _single-phase-initialization:
+
+Legacy single-phase initialization
+..................................
+
+.. attention::
+   Single-phase initialization is a legacy mechanism to initialize extension
+   modules, with known drawbacks and design flaws. Extension module authors
+   are encouraged to use multi-phase initialization instead.
+
+In single-phase initialization, the
+:ref:`initialization function <extension-export-hook>` (``PyInit_modulename``)
+should create, populate and return a module object.
+This is typically done using :c:func:`PyModule_Create` and functions like
+:c:func:`PyModule_AddObjectRef`.
+
+Single-phase initialization differs from the :ref:`default <multi-phase-initialization>`
+in the following ways:
+
+* Single-phase modules are, or rather *contain*, “singletons”.
+
+  When the module is first initialized, Python saves the contents of
+  the module's ``__dict__`` (that is, typically, the module's functions and
+  types).
+
+  For subsequent imports, Python does not call the initialization function
+  again.
+  Instead, it creates a new module object with a new ``__dict__``, and copies
+  the saved contents to it.
+  For example, given a single-phase module ``_testsinglephase``
+  [#testsinglephase]_ that defines a function ``sum`` and an exception class
+  ``error``:
+
+  .. code-block:: python
+
+     >>> import sys
+     >>> import _testsinglephase as one
+     >>> del sys.modules['_testsinglephase']
+     >>> import _testsinglephase as two
+     >>> one is two
+     False
+     >>> one.__dict__ is two.__dict__
+     False
+     >>> one.sum is two.sum
+     True
+     >>> one.error is two.error
+     True
+
+  The exact behavior should be considered a CPython implementation detail.
+
+* To work around the fact that ``PyInit_modulename`` does not take a *spec*
+  argument, some state of the import machinery is saved and applied to the
+  first suitable module created during the ``PyInit_modulename`` call.
+  Specifically, when a sub-module is imported, this mechanism prepends the
+  parent package name to the name of the module.
+
+  A single-phase ``PyInit_modulename`` function should create “its” module
+  object as soon as possible, before any other module objects can be created.
+
+* Non-ASCII module names (``PyInitU_modulename``) are not supported.
+
+* Single-phase modules support module lookup functions like
+  :c:func:`PyState_FindModule`.
+
+.. [#testsinglephase] ``_testsinglephase`` is an internal module used
+   in CPython's self-test suite; your installation may or may not
+   include it.

Doc/c-api/file.rst

@@ -93,6 +93,29 @@ the :mod:`io` APIs instead.
    .. versionadded:: 3.8
 
 
+.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path)
+
+   Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str`
+   object. The behavior of this function may be overridden by
+   :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the
+   text.
+
+   This is analogous to :func:`io.open_code` in Python.
+
+   On success, this function returns a :term:`strong reference` to a Python
+   file object. On failure, this function returns ``NULL`` with an exception
+   set.
+
+   .. versionadded:: 3.8
+
+
+.. c:function:: PyObject *PyFile_OpenCode(const char *path)
+
+   Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a
+   UTF-8 encoded :c:expr:`const char*`.
+
+   .. versionadded:: 3.8
+
 
 .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags)
 

Doc/c-api/float.rst

@@ -78,6 +78,104 @@ Floating-Point Objects
    Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`.
 
 
+.. c:macro:: Py_INFINITY
+
+   This macro expands a to constant expression of type :c:expr:`double`, that
+   represents the positive infinity.
+
+   On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from
+   the C11 standard ``<math.h>`` header.
+
+
+.. c:macro:: Py_NAN
+
+   This macro expands a to constant expression of type :c:expr:`double`, that
+   represents a quiet not-a-number (qNaN) value.
+
+   On most platforms, this is equivalent to the :c:macro:`!NAN` macro from
+   the C11 standard ``<math.h>`` header.
+
+
+.. c:macro:: Py_HUGE_VAL
+
+   Equivalent to :c:macro:`!INFINITY`.
+
+   .. deprecated:: 3.14
+      The macro is :term:`soft deprecated`.
+
+
+.. c:macro:: Py_MATH_E
+
+   The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant.
+
+
+.. c:macro:: Py_MATH_El
+
+   High precision (long double) definition of :data:`~math.e` constant.
+
+
+.. c:macro:: Py_MATH_PI
+
+   The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant.
+
+
+.. c:macro:: Py_MATH_PIl
+
+   High precision (long double) definition of :data:`~math.pi` constant.
+
+
+.. c:macro:: Py_MATH_TAU
+
+   The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant.
+
+   .. versionadded:: 3.6
+
+
+.. c:macro:: Py_RETURN_NAN
+
+   Return :data:`math.nan` from a function.
+
+   On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``.
+
+
+.. c:macro:: Py_RETURN_INF(sign)
+
+   Return :data:`math.inf` or :data:`-math.inf <math.inf>` from a function,
+   depending on the sign of *sign*.
+
+   On most platforms, this is equivalent to the following::
+
+      return PyFloat_FromDouble(copysign(INFINITY, sign));
+
+
+.. c:macro:: Py_IS_FINITE(X)
+
+   Return ``1`` if the given floating-point number *X* is finite,
+   that is, it is normal, subnormal or zero, but not infinite or NaN.
+   Return ``0`` otherwise.
+
+   .. deprecated:: 3.14
+      The macro is :term:`soft deprecated`.  Use :c:macro:`!isfinite` instead.
+
+
+.. c:macro:: Py_IS_INFINITY(X)
+
+   Return ``1`` if the given floating-point number *X* is positive or negative
+   infinity.  Return ``0`` otherwise.
+
+   .. deprecated:: 3.14
+      The macro is :term:`soft deprecated`.  Use :c:macro:`!isinf` instead.
+
+
+.. c:macro:: Py_IS_NAN(X)
+
+   Return ``1`` if the given floating-point number *X* is a not-a-number (NaN)
+   value.  Return ``0`` otherwise.
+
+   .. deprecated:: 3.14
+      The macro is :term:`soft deprecated`.  Use :c:macro:`!isnan` instead.
+
+
 Pack and Unpack functions
 -------------------------
 
@@ -96,8 +194,8 @@ NaNs (if such things exist on the platform) isn't handled correctly, and
 attempting to unpack a bytes string containing an IEEE INF or NaN will raise an
 exception.
 
-Note that NaNs type may not be preserved on IEEE platforms (silent NaN become
-quiet), for example on x86 systems in 32-bit mode.
+Note that NaNs type may not be preserved on IEEE platforms (signaling NaN become
+quiet NaN), for example on x86 systems in 32-bit mode.
 
 On non-IEEE platforms with more precision, or larger dynamic range, than IEEE
 754 supports, not all values can be packed; on non-IEEE platforms with less
@@ -124,15 +222,15 @@ There are two problems on non-IEEE platforms:
 * What this does is undefined if *x* is a NaN or infinity.
 * ``-0.0`` and ``+0.0`` produce the same bytes string.
 
-.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le)
+.. c:function:: int PyFloat_Pack2(double x, char *p, int le)
 
    Pack a C double as the IEEE 754 binary16 half-precision format.
 
-.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le)
+.. c:function:: int PyFloat_Pack4(double x, char *p, int le)
 
    Pack a C double as the IEEE 754 binary32 single precision format.
 
-.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le)
+.. c:function:: int PyFloat_Pack8(double x, char *p, int le)
 
    Pack a C double as the IEEE 754 binary64 double precision format.
 
@@ -154,14 +252,14 @@ Return value: The unpacked double.  On error, this is ``-1.0`` and
 Note that on a non-IEEE platform this will refuse to unpack a bytes string that
 represents a NaN or infinity.
 
-.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le)
+.. c:function:: double PyFloat_Unpack2(const char *p, int le)
 
    Unpack the IEEE 754 binary16 half-precision format as a C double.
 
-.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le)
+.. c:function:: double PyFloat_Unpack4(const char *p, int le)
 
    Unpack the IEEE 754 binary32 single precision format as a C double.
 
-.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le)
+.. c:function:: double PyFloat_Unpack8(const char *p, int le)
 
    Unpack the IEEE 754 binary64 double precision format as a C double.

Doc/c-api/frame.rst

@@ -29,6 +29,12 @@ See also :ref:`Reflection <reflection>`.
       Previously, this type was only available after including
       ``<frameobject.h>``.
 
+.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals)
+
+   Create a new frame object. This function returns a :term:`strong reference`
+   to the new frame object on success, and returns ``NULL`` with an exception
+   set on failure.
+
 .. c:function:: int PyFrame_Check(PyObject *obj)
 
    Return non-zero if *obj* is a frame object.
@@ -161,6 +167,57 @@ See :pep:`667` for more information.
 
    Return non-zero if *obj* is a frame :func:`locals` proxy.
 
+
+Legacy Local Variable APIs
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing.
+They exist solely for backwards compatibility.
+
+
+.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear)
+
+   This function is :term:`soft deprecated` and does nothing.
+
+   Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals`
+   attribute of *f* to the internal "fast" array of local variables, allowing
+   changes in frame objects to be visible to the interpreter. If *clear* was
+   true, this function would process variables that were unset in the locals
+   dictionary.
+
+   .. versionchanged:: 3.13
+      This function now does nothing.
+
+
+.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f)
+
+   This function is :term:`soft deprecated` and does nothing.
+
+   Prior to Python 3.13, this function would copy the internal "fast" array
+   of local variables (which is used by the interpreter) to the
+   :attr:`~frame.f_locals` attribute of *f*, allowing changes in local
+   variables to be visible to frame objects.
+
+   .. versionchanged:: 3.13
+      This function now does nothing.
+
+
+.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f)
+
+   This function is :term:`soft deprecated` and does nothing.
+
+   Prior to Python 3.13, this function was similar to
+   :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and
+   ``-1`` with an exception set on failure.
+
+   .. versionchanged:: 3.13
+      This function now does nothing.
+
+
+.. seealso::
+   :pep:`667`
+
+
 Internal Frames
 ^^^^^^^^^^^^^^^
 

Doc/c-api/function.rst

@@ -95,6 +95,22 @@ There are a few functions specific to Python functions.
 
    .. versionadded:: 3.12
 
+
+.. c:function:: PyObject* PyFunction_GetKwDefaults(PyObject *op)
+
+   Return the keyword-only argument default values of the function object *op*. This can be a
+   dictionary of arguments or ``NULL``.
+
+
+.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults)
+
+   Set the keyword-only argument default values of the function object *op*.
+   *defaults* must be a dictionary of keyword-only arguments or ``Py_None``.
+
+   This function returns ``0`` on success, and returns ``-1`` with an exception
+   set on failure.
+
+
 .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op)
 
    Return the closure associated with the function object *op*. This can be ``NULL``
@@ -123,6 +139,19 @@ There are a few functions specific to Python functions.
    Raises :exc:`SystemError` and returns ``-1`` on failure.
 
 
+.. c:function:: PyObject *PyFunction_GET_CODE(PyObject *op)
+                PyObject *PyFunction_GET_GLOBALS(PyObject *op)
+                PyObject *PyFunction_GET_MODULE(PyObject *op)
+                PyObject *PyFunction_GET_DEFAULTS(PyObject *op)
+                PyObject *PyFunction_GET_KW_DEFAULTS(PyObject *op)
+                PyObject *PyFunction_GET_CLOSURE(PyObject *op)
+                PyObject *PyFunction_GET_ANNOTATIONS(PyObject *op)
+
+   These functions are similar to their ``PyFunction_Get*`` counterparts, but
+   do not do type checking. Passing anything other than an instance of
+   :c:data:`PyFunction_Type` is undefined behavior.
+
+
 .. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback)
 
    Register *callback* as a function watcher for the current interpreter.
@@ -169,7 +198,7 @@ There are a few functions specific to Python functions.
    unpredictable effects, including infinite recursion.
 
    If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked
-   after `func` has been fully initialized. Otherwise, the callback is invoked
+   after *func* has been fully initialized. Otherwise, the callback is invoked
    before the modification to *func* takes place, so the prior state of *func*
    can be inspected. The runtime is permitted to optimize away the creation of
    function objects when possible. In such cases no event will be emitted.

Doc/c-api/gcsupport.rst

@@ -57,11 +57,49 @@ rules:
    Analogous to :c:macro:`PyObject_New` but for container objects with the
    :c:macro:`Py_TPFLAGS_HAVE_GC` flag set.
 
+   Do not call this directly to allocate memory for an object; call the type's
+   :c:member:`~PyTypeObject.tp_alloc` slot instead.
+
+   When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot,
+   :c:func:`PyType_GenericAlloc` is preferred over a custom function that
+   simply calls this macro.
+
+   Memory allocated by this macro must be freed with
+   :c:func:`PyObject_GC_Del` (usually called via the object's
+   :c:member:`~PyTypeObject.tp_free` slot).
+
+   .. seealso::
+
+      * :c:func:`PyObject_GC_Del`
+      * :c:macro:`PyObject_New`
+      * :c:func:`PyType_GenericAlloc`
+      * :c:member:`~PyTypeObject.tp_alloc`
+
+
 .. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size)
 
    Analogous to :c:macro:`PyObject_NewVar` but for container objects with the
    :c:macro:`Py_TPFLAGS_HAVE_GC` flag set.
 
+   Do not call this directly to allocate memory for an object; call the type's
+   :c:member:`~PyTypeObject.tp_alloc` slot instead.
+
+   When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot,
+   :c:func:`PyType_GenericAlloc` is preferred over a custom function that
+   simply calls this macro.
+
+   Memory allocated by this macro must be freed with
+   :c:func:`PyObject_GC_Del` (usually called via the object's
+   :c:member:`~PyTypeObject.tp_free` slot).
+
+   .. seealso::
+
+      * :c:func:`PyObject_GC_Del`
+      * :c:macro:`PyObject_NewVar`
+      * :c:func:`PyType_GenericAlloc`
+      * :c:member:`~PyTypeObject.tp_alloc`
+
+
 .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size)
 
    Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size*
@@ -73,6 +111,10 @@ rules:
    The extra data will be deallocated with the object, but otherwise it is
    not managed by Python.
 
+   Memory allocated by this function must be freed with
+   :c:func:`PyObject_GC_Del` (usually called via the object's
+   :c:member:`~PyTypeObject.tp_free` slot).
+
    .. warning::
       The function is marked as unstable because the final mechanism
       for reserving extra data after an instance is not yet decided.
@@ -136,6 +178,21 @@ rules:
    Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or
    :c:macro:`PyObject_GC_NewVar`.
 
+   Do not call this directly to free an object's memory; call the type's
+   :c:member:`~PyTypeObject.tp_free` slot instead.
+
+   Do not use this for memory allocated by :c:macro:`PyObject_New`,
+   :c:macro:`PyObject_NewVar`, or related allocation functions; use
+   :c:func:`PyObject_Free` instead.
+
+   .. seealso::
+
+      * :c:func:`PyObject_Free` is the non-GC equivalent of this function.
+      * :c:macro:`PyObject_GC_New`
+      * :c:macro:`PyObject_GC_NewVar`
+      * :c:func:`PyType_GenericAlloc`
+      * :c:member:`~PyTypeObject.tp_free`
+
 
 .. c:function:: void PyObject_GC_UnTrack(void *op)
 
@@ -180,9 +237,9 @@ provided.  In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse`
 must name its arguments exactly *visit* and *arg*:
 
 
-.. c:function:: void Py_VISIT(PyObject *o)
+.. c:macro:: Py_VISIT(o)
 
-   If *o* is not ``NULL``, call the *visit* callback, with arguments *o*
+   If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o*
    and *arg*.  If *visit* returns a non-zero value, then return it.
    Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers
    look like::

Doc/c-api/gen.rst

@@ -44,3 +44,41 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`.
    with ``__name__`` and ``__qualname__`` set to *name* and *qualname*.
    A reference to *frame* is stolen by this function.  The *frame* argument
    must not be ``NULL``.
+
+.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen)
+
+   Return a new :term:`strong reference` to the code object wrapped by *gen*.
+   This function always succeeds.
+
+
+Asynchronous Generator Objects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. seealso::
+   :pep:`525`
+
+.. c:var:: PyTypeObject PyAsyncGen_Type
+
+   The type object corresponding to asynchronous generator objects. This is
+   available as :class:`types.AsyncGeneratorType` in the Python layer.
+
+   .. versionadded:: 3.6
+
+.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname)
+
+   Create a new asynchronous generator wrapping *frame*, with ``__name__`` and
+   ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this
+   function and must not be ``NULL``.
+
+   On success, this function returns a :term:`strong reference` to the
+   new asynchronous generator. On failure, this function returns ``NULL``
+   with an exception set.
+
+   .. versionadded:: 3.6
+
+.. c:function:: int PyAsyncGen_CheckExact(PyObject *op)
+
+   Return true if *op* is an asynchronous generator object, false otherwise.
+   This function always succeeds.
+
+   .. versionadded:: 3.6

Doc/c-api/hash.rst

@@ -11,47 +11,103 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`.
 
    .. versionadded:: 3.2
 
+
 .. c:type:: Py_uhash_t
 
    Hash value type: unsigned integer.
 
    .. versionadded:: 3.2
 
+
+.. c:macro:: Py_HASH_ALGORITHM
+
+   A numerical value indicating the algorithm for hashing of :class:`str`,
+   :class:`bytes`, and :class:`memoryview`.
+
+   The algorithm name is exposed by :data:`sys.hash_info.algorithm`.
+
+   .. versionadded:: 3.4
+
+
+.. c:macro:: Py_HASH_FNV
+             Py_HASH_SIPHASH24
+             Py_HASH_SIPHASH13
+
+   Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine
+   which algorithm is used for hashing. The hash algorithm can be configured
+   via the configure :option:`--with-hash-algorithm` option.
+
+   .. versionadded:: 3.4
+      Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`.
+
+   .. versionadded:: 3.11
+      Add :c:macro:`!Py_HASH_SIPHASH13`.
+
+
+.. c:macro:: Py_HASH_CUTOFF
+
+   Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A
+   instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`.
+
+   - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization.
+   - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7.
+
+   32-bit platforms should use a cutoff smaller than 64-bit platforms because
+   it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms
+   and 5 on 32-bit platforms should provide a decent safety margin.
+
+   This corresponds to the :data:`sys.hash_info.cutoff` constant.
+
+   .. versionadded:: 3.4
+
+
 .. c:macro:: PyHASH_MODULUS
 
-   The `Mersenne prime <https://en.wikipedia.org/wiki/Mersenne_prime>`_ ``P = 2**n -1``, used for numeric hash scheme.
+   The `Mersenne prime <https://en.wikipedia.org/wiki/Mersenne_prime>`_ ``P = 2**n -1``,
+   used for numeric hash scheme.
+
+   This corresponds to the :data:`sys.hash_info.modulus` constant.
 
    .. versionadded:: 3.13
 
+
 .. c:macro:: PyHASH_BITS
 
    The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`.
 
    .. versionadded:: 3.13
 
+
 .. c:macro:: PyHASH_MULTIPLIER
 
    Prime multiplier used in string and various other hashes.
 
    .. versionadded:: 3.13
 
+
 .. c:macro:: PyHASH_INF
 
    The hash value returned for a positive infinity.
 
+   This corresponds to the :data:`sys.hash_info.inf` constant.
+
    .. versionadded:: 3.13
 
+
 .. c:macro:: PyHASH_IMAG
 
    The multiplier used for the imaginary part of a complex number.
 
+   This corresponds to the :data:`sys.hash_info.imag` constant.
+
    .. versionadded:: 3.13
 
+
 .. c:type:: PyHash_FuncDef
 
    Hash function definition used by :c:func:`PyHash_GetFuncDef`.
 
-   .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t)
+   .. c:member:: Py_hash_t (*const hash)(const void *, Py_ssize_t)
 
       Hash function.
 
@@ -59,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`.
 
       Hash function name (UTF-8 encoded string).
 
+      This corresponds to the :data:`sys.hash_info.algorithm` constant.
+
    .. c:member:: const int hash_bits
 
       Internal size of the hash value in bits.
 
+      This corresponds to the :data:`sys.hash_info.hash_bits` constant.
+
    .. c:member:: const int seed_bits
 
       Size of seed input in bits.
 
+      This corresponds to the :data:`sys.hash_info.seed_bits` constant.
+
    .. versionadded:: 3.4
 
 

Doc/c-api/import.rst

@@ -327,6 +327,13 @@ Importing Modules
    initialization.
 
 
+.. c:var:: struct _inittab *PyImport_Inittab
+
+   The table of built-in modules used by Python initialization. Do not use this directly;
+   use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab`
+   instead.
+
+
 .. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name)
 
    Import the module *mod_name* and get its attribute *attr_name*.

Doc/c-api/index.rst

@@ -17,6 +17,7 @@ document the API functions in detail.
    veryhigh.rst
    refcounting.rst
    exceptions.rst
+   extension-modules.rst
    utilities.rst
    abstract.rst
    concrete.rst