Merge pull request #104080 from Repiteo/ci/cache-trim

CI: Trim cache before saving

.github/actions/godot-build/action.yml

@@ -19,11 +19,6 @@ inputs:
   scons-cache:
     description: The SCons cache path.
     default: ${{ github.workspace }}/.scons_cache/
-  scons-cache-limit:
-    description: The SCons cache size limit.
-    # actions/cache has 10 GiB limit, and GitHub runners have a 14 GiB disk.
-    # Limit to 7 GiB to avoid having the extracted cache fill the disk.
-    default: 7
 
 runs:
   using: composite
@@ -33,7 +28,7 @@ runs:
       env:
         SCONSFLAGS: ${{ inputs.sconsflags }}
       run: |
-        echo "Building with flags:" platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}" cache_limit=${{ inputs.scons-cache-limit }}
+        echo "Building with flags:" platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}"
 
         if [ "${{ inputs.target }}" != "editor" ]; then
           # Ensure we don't include editor code in export template builds.
@@ -47,5 +42,5 @@ runs:
           export BUILD_NAME="gh"
         fi
 
-        scons platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}" cache_limit=${{ inputs.scons-cache-limit }}
+        scons platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}"
         ls -l bin/

.github/actions/godot-cache-restore/action.yml

@@ -11,42 +11,29 @@ inputs:
 runs:
   using: composite
   steps:
-    # Because all branches can refer to the repository's default branch's cache, we want it to
+    - name: Restore default cache
-    # persist as the de-facto fallback. However, it easily expunges in a matter of hours if
-    # nothing explicitly calls to it, so we work around that by ensuring it's *always* pinged
-    # prior to any cache operations.
-    - name: Ping main cache
       uses: actions/cache/restore@v4
       id: cache-ping
       with:
         path: ${{ inputs.scons-cache }}
-        key: ${{ github.sha }} # Dummy key; we have to rely on the fallback value.
+        key: ${{ inputs.cache-name }}|${{ github.event.repository.default_branch }}|${{ github.sha }}
         restore-keys: ${{ inputs.cache-name }}|${{ github.event.repository.default_branch }}
-        lookup-only: true
 
-    # Fallback access isn't logged, so register an explicit cache-hit if found.
+    - name: Log default cache files
-    - name: Ping main cache (exact)
+      if: github.ref_name != github.event.repository.default_branch
-      if: steps.cache-ping.outputs.cache-matched-key
+      shell: sh
-      uses: actions/cache/restore@v4
+      run: find "${{ inputs.scons-cache }}" >> redundant.txt
-      with:
-        path: ${{ inputs.scons-cache }}
-        key: ${{ steps.cache-ping.outputs.cache-matched-key }}
-        lookup-only: true
 
-    # We try to match an existing cache to restore from it. Each potential key is checked against
+    # This is done after pulling the default cache so that PRs can integrate any potential changes
-    # all existing caches as a prefix. E.g. 'linux-template-minimal' would match any cache that
+    # from the default branch without conflicting with whatever local changes were already made.
-    # starts with "linux-template-minimal", such as
+    - name: Restore local cache
-    # "linux-template-minimal|master|6588a4a29af1621086feac0117d5d4d37af957fd".
-    #
-    # We check these prefixes in this order:
-    #   1. An exact match for the base branch, reference name, and SHA hash.
-    #   2. A partial match for the same cache name and reference name.
-    #   3. A partial match for the same cache name and default branch name.
-    - name: Restore SCons cache directory
       uses: actions/cache/restore@v4
+      if: github.ref_name != github.event.repository.default_branch
       with:
         path: ${{ inputs.scons-cache }}
         key: ${{ inputs.cache-name }}|${{ github.ref_name }}|${{ github.sha }}
-        restore-keys: |
+        restore-keys: ${{ inputs.cache-name }}|${{ github.ref_name }}
-          ${{ inputs.cache-name }}|${{ github.ref_name }}
+
-          ${{ inputs.cache-name }}|${{ github.event.repository.default_branch }}
+    - name: Store unix timestamp
+      shell: sh
+      run: echo "CACHE_TIMESTAMP=$(date +%s)" >> $GITHUB_ENV

.github/actions/godot-cache-save/action.yml

@@ -11,6 +11,10 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Purge files before timestamp
+      shell: sh
+      run: misc/scripts/purge_cache.py ${{ env.CACHE_TIMESTAMP }} "${{ inputs.scons-cache }}"
+
     - name: Save SCons cache directory
       uses: actions/cache/save@v4
       with:

.github/workflows/android_builds.yml

@@ -20,21 +20,18 @@ jobs:
             target: editor
             tests: false
             sconsflags: arch=arm64 production=yes swappy=yes
-            cache-limit: 1
 
           - name: Template arm32 (target=template_release, arch=arm32)
             cache-name: android-template-arm32
             target: template_release
             tests: false
             sconsflags: arch=arm32 swappy=yes
-            cache-limit: 1
 
           - name: Template arm64 (target=template_release, arch=arm64)
             cache-name: android-template-arm64
             target: template_release
             tests: false
             sconsflags: arch=arm64 swappy=yes
-            cache-limit: 1
 
     steps:
       - name: Checkout
@@ -75,7 +72,6 @@ jobs:
           platform: android
           target: ${{ matrix.target }}
           tests: ${{ matrix.tests }}
-          scons-cache-limit: ${{ matrix.cache-limit }}
 
       - name: Save Godot build cache
         uses: ./.github/actions/godot-cache-save

.github/workflows/ios_builds.yml

@@ -32,7 +32,6 @@ jobs:
           platform: ios
           target: template_release
           tests: false
-          scons-cache-limit: 1
 
       - name: Save Godot build cache
         uses: ./.github/actions/godot-cache-save

.github/workflows/linux_builds.yml

@@ -34,7 +34,6 @@ jobs:
             artifact: true
             # Validate godot-cpp compatibility on one arbitrary editor build.
             godot-cpp: true
-            cache-limit: 2
 
           - name: Editor with doubles and GCC sanitizers (target=editor, tests=yes, dev_build=yes, scu_build=yes, precision=double, use_asan=yes, use_ubsan=yes, linker=gold)
             cache-name: linux-editor-double-sanitizers
@@ -47,7 +46,6 @@ jobs:
             proj-test: true
             # Skip 2GiB artifact speeding up action.
             artifact: false
-            cache-limit: 7
 
           - name: Editor with clang sanitizers (target=editor, tests=yes, dev_build=yes, use_asan=yes, use_ubsan=yes, use_llvm=yes, linker=lld)
             cache-name: linux-editor-llvm-sanitizers
@@ -60,7 +58,6 @@ jobs:
             artifact: false
             # Test our oldest supported SCons/Python versions on one arbitrary editor build.
             legacy-scons: true
-            cache-limit: 7
 
           - name: Editor with ThreadSanitizer (target=editor, tests=yes, dev_build=yes, use_tsan=yes, use_llvm=yes, linker=lld)
             cache-name: linux-editor-thread-sanitizer
@@ -71,7 +68,6 @@ jobs:
             build-mono: false
             # Skip 2GiB artifact speeding up action.
             artifact: false
-            cache-limit: 5
 
           - name: Template w/ Mono, release (target=template_release, tests=yes)
             cache-name: linux-template-mono
@@ -81,7 +77,6 @@ jobs:
             build-mono: false
             tests: true
             artifact: true
-            cache-limit: 1
 
           - name: Template w/ Mono, debug (target=template_debug, tests=yes)
             cache-name: linux-template-mono-debug
@@ -91,7 +86,6 @@ jobs:
             build-mono: false
             tests: true
             artifact: true
-            cache-limit: 1
 
           - name: Minimal template (target=template_release, tests=yes, everything disabled)
             cache-name: linux-template-minimal
@@ -100,7 +94,6 @@ jobs:
             bin: ./bin/godot.linuxbsd.template_release.x86_64
             tests: true
             artifact: true
-            cache-limit: 1
 
     steps:
       - name: Checkout
@@ -165,7 +158,6 @@ jobs:
           platform: linuxbsd
           target: ${{ matrix.target }}
           tests: ${{ matrix.tests }}
-          scons-cache-limit: ${{ matrix.cache-limit }}
 
       - name: Compilation (godot-cpp)
         uses: ./.github/actions/godot-cpp-build

.github/workflows/macos_builds.yml

@@ -20,7 +20,6 @@ jobs:
             target: editor
             tests: true
             bin: ./bin/godot.macos.editor.universal
-            cache-limit: 1
 
           - name: Template (target=template_release, tests=yes)
             cache-name: macos-template
@@ -28,7 +27,6 @@ jobs:
             tests: true
             sconsflags: debug_symbols=no
             bin: ./bin/godot.macos.template_release.universal
-            cache-limit: 1
 
     steps:
       - name: Checkout
@@ -56,7 +54,6 @@ jobs:
           platform: macos
           target: ${{ matrix.target }}
           tests: ${{ matrix.tests }}
-          scons-cache-limit: 0 # Only cap on second run to avoid purging unnecessarily
 
       - name: Compilation (arm64)
         uses: ./.github/actions/godot-build
@@ -65,7 +62,6 @@ jobs:
           platform: macos
           target: ${{ matrix.target }}
           tests: ${{ matrix.tests }}
-          scons-cache-limit: ${{ matrix.cache-limit }}
 
       - name: Save Godot build cache
         uses: ./.github/actions/godot-cache-save

.github/workflows/web_builds.yml

@@ -62,7 +62,6 @@ jobs:
           platform: web
           target: ${{ matrix.target }}
           tests: ${{ matrix.tests }}
-          scons-cache-limit: 0.5
 
       - name: Save Godot build cache
         uses: ./.github/actions/godot-cache-save

.github/workflows/windows_builds.yml

@@ -26,7 +26,6 @@ jobs:
             sconsflags: debug_symbols=no vsproj=yes vsproj_gen_only=no windows_subsystem=console
             bin: ./bin/godot.windows.editor.x86_64.exe
             compiler: msvc
-            cache-limit: 2
 
           - name: Editor w/ clang-cl (target=editor, tests=yes, use_llvm=yes)
             cache-name: windows-editor-clang
@@ -35,7 +34,6 @@ jobs:
             sconsflags: debug_symbols=no windows_subsystem=console use_llvm=yes
             bin: ./bin/godot.windows.editor.x86_64.llvm.exe
             compiler: clang
-            cache-limit: 1
 
           - name: Template (target=template_release, tests=yes)
             cache-name: windows-template
@@ -44,7 +42,6 @@ jobs:
             sconsflags: debug_symbols=no
             bin: ./bin/godot.windows.template_release.x86_64.console.exe
             compiler: msvc
-            cache-limit: 2
 
           - name: Template w/ GCC (target=template_release, tests=yes, use_mingw=yes)
             cache-name: windows-template-gcc
@@ -54,7 +51,6 @@ jobs:
             sconsflags: debug_symbols=no use_mingw=yes
             bin: ./bin/godot.windows.template_release.x86_64.console.exe
             compiler: gcc
-            cache-limit: 1
 
     steps:
       - name: Checkout
@@ -92,7 +88,6 @@ jobs:
           platform: windows
           target: ${{ matrix.target }}
           tests: ${{ matrix.tests }}
-          scons-cache-limit: ${{ matrix.cache-limit }}
 
       - name: Save Godot build cache
         uses: ./.github/actions/godot-cache-save

methods.py

@@ -897,11 +897,9 @@ def prepare_cache(env) -> None:
     # Convert GiB to bytes; treat negative numbers as 0 (unlimited).
     cache_limit = max(0, int(cache_limit * 1024 * 1024 * 1024))
     if env["verbose"]:
-        print(
+        print_info(
-            "Current cache limit is {} (used: {})".format(
+            f"Current cache size is {convert_size(get_size(cache_path))}"
-                convert_size(cache_limit) if cache_limit else "∞",
+            + (f" (limit: {convert_size(cache_limit)})" if cache_limit else "")
-                convert_size(get_size(cache_path)),
-            )
         )
 
     atexit.register(clean_cache, cache_path, cache_limit, env["verbose"])

misc/scripts/purge_cache.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import argparse
+import glob
+import os
+
+if __name__ != "__main__":
+    raise ImportError(f"{__name__} should not be used as a module.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Cleanup old cache files")
+    parser.add_argument("timestamp", type=int, help="Unix timestamp cutoff")
+    parser.add_argument("directory", help="Path to cache directory")
+    args = parser.parse_args()
+
+    ret = 0
+
+    # TODO: Convert to non-hardcoded path
+    if os.path.exists("redundant.txt"):
+        with open("redundant.txt") as redundant:
+            for item in map(str.strip, redundant):
+                if os.path.isfile(item):
+                    try:
+                        os.remove(item)
+                    except OSError:
+                        print(f'Failed to handle "{item}"; skipping.')
+                        ret += 1
+
+    for file in glob.glob(os.path.join(args.directory, "*", "*")):
+        try:
+            if os.path.getatime(file) < args.timestamp:
+                os.remove(file)
+        except OSError:
+            print(f'Failed to handle "{file}"; skipping.')
+            ret += 1
+
+    return ret
+
+
+try:
+    raise SystemExit(main())
+except KeyboardInterrupt:
+    import signal
+
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    os.kill(os.getpid(), signal.SIGINT)