diff --git a/.github/actions/godot-build/action.yml b/.github/actions/godot-build/action.yml index 2405cc22be2..4d393a8c112 100644 --- a/.github/actions/godot-build/action.yml +++ b/.github/actions/godot-build/action.yml @@ -19,11 +19,6 @@ inputs: scons-cache: description: The SCons cache path. default: ${{ github.workspace }}/.scons_cache/ - scons-cache-limit: - description: The SCons cache size limit. - # actions/cache has 10 GiB limit, and GitHub runners have a 14 GiB disk. - # Limit to 7 GiB to avoid having the extracted cache fill the disk. - default: 7 runs: using: composite @@ -33,7 +28,7 @@ runs: env: SCONSFLAGS: ${{ inputs.sconsflags }} run: | - echo "Building with flags:" platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}" cache_limit=${{ inputs.scons-cache-limit }} + echo "Building with flags:" platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}" if [ "${{ inputs.target }}" != "editor" ]; then # Ensure we don't include editor code in export template builds. @@ -47,5 +42,5 @@ runs: export BUILD_NAME="gh" fi - scons platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}" cache_limit=${{ inputs.scons-cache-limit }} + scons platform=${{ inputs.platform }} target=${{ inputs.target }} tests=${{ inputs.tests }} ${{ env.SCONSFLAGS }} "cache_path=${{ inputs.scons-cache }}" ls -l bin/ diff --git a/.github/actions/godot-cache-restore/action.yml b/.github/actions/godot-cache-restore/action.yml index 0b3687fb99c..335965f931f 100644 --- a/.github/actions/godot-cache-restore/action.yml +++ b/.github/actions/godot-cache-restore/action.yml @@ -11,42 +11,29 @@ inputs: runs: using: composite steps: - # Because all branches can refer to the repository's default branch's cache, we want it to - # persist as the de-facto fallback. However, it easily expunges in a matter of hours if - # nothing explicitly calls to it, so we work around that by ensuring it's *always* pinged - # prior to any cache operations. - - name: Ping main cache + - name: Restore default cache uses: actions/cache/restore@v4 id: cache-ping with: path: ${{ inputs.scons-cache }} - key: ${{ github.sha }} # Dummy key; we have to rely on the fallback value. + key: ${{ inputs.cache-name }}|${{ github.event.repository.default_branch }}|${{ github.sha }} restore-keys: ${{ inputs.cache-name }}|${{ github.event.repository.default_branch }} - lookup-only: true - # Fallback access isn't logged, so register an explicit cache-hit if found. - - name: Ping main cache (exact) - if: steps.cache-ping.outputs.cache-matched-key - uses: actions/cache/restore@v4 - with: - path: ${{ inputs.scons-cache }} - key: ${{ steps.cache-ping.outputs.cache-matched-key }} - lookup-only: true + - name: Log default cache files + if: github.ref_name != github.event.repository.default_branch + shell: sh + run: find "${{ inputs.scons-cache }}" >> redundant.txt - # We try to match an existing cache to restore from it. Each potential key is checked against - # all existing caches as a prefix. E.g. 'linux-template-minimal' would match any cache that - # starts with "linux-template-minimal", such as - # "linux-template-minimal|master|6588a4a29af1621086feac0117d5d4d37af957fd". - # - # We check these prefixes in this order: - # 1. An exact match for the base branch, reference name, and SHA hash. - # 2. A partial match for the same cache name and reference name. - # 3. A partial match for the same cache name and default branch name. - - name: Restore SCons cache directory + # This is done after pulling the default cache so that PRs can integrate any potential changes + # from the default branch without conflicting with whatever local changes were already made. + - name: Restore local cache uses: actions/cache/restore@v4 + if: github.ref_name != github.event.repository.default_branch with: path: ${{ inputs.scons-cache }} key: ${{ inputs.cache-name }}|${{ github.ref_name }}|${{ github.sha }} - restore-keys: | - ${{ inputs.cache-name }}|${{ github.ref_name }} - ${{ inputs.cache-name }}|${{ github.event.repository.default_branch }} + restore-keys: ${{ inputs.cache-name }}|${{ github.ref_name }} + + - name: Store unix timestamp + shell: sh + run: echo "CACHE_TIMESTAMP=$(date +%s)" >> $GITHUB_ENV diff --git a/.github/actions/godot-cache-save/action.yml b/.github/actions/godot-cache-save/action.yml index 6ec735a18f8..cc7dc42965e 100644 --- a/.github/actions/godot-cache-save/action.yml +++ b/.github/actions/godot-cache-save/action.yml @@ -11,6 +11,10 @@ inputs: runs: using: composite steps: + - name: Purge files before timestamp + shell: sh + run: misc/scripts/purge_cache.py ${{ env.CACHE_TIMESTAMP }} "${{ inputs.scons-cache }}" + - name: Save SCons cache directory uses: actions/cache/save@v4 with: diff --git a/.github/workflows/android_builds.yml b/.github/workflows/android_builds.yml index a489bc1393c..da8a1bc16d0 100644 --- a/.github/workflows/android_builds.yml +++ b/.github/workflows/android_builds.yml @@ -20,21 +20,18 @@ jobs: target: editor tests: false sconsflags: arch=arm64 production=yes swappy=yes - cache-limit: 1 - name: Template arm32 (target=template_release, arch=arm32) cache-name: android-template-arm32 target: template_release tests: false sconsflags: arch=arm32 swappy=yes - cache-limit: 1 - name: Template arm64 (target=template_release, arch=arm64) cache-name: android-template-arm64 target: template_release tests: false sconsflags: arch=arm64 swappy=yes - cache-limit: 1 steps: - name: Checkout @@ -75,7 +72,6 @@ jobs: platform: android target: ${{ matrix.target }} tests: ${{ matrix.tests }} - scons-cache-limit: ${{ matrix.cache-limit }} - name: Save Godot build cache uses: ./.github/actions/godot-cache-save diff --git a/.github/workflows/ios_builds.yml b/.github/workflows/ios_builds.yml index 3e79df4fcb9..d4722e7c0d7 100644 --- a/.github/workflows/ios_builds.yml +++ b/.github/workflows/ios_builds.yml @@ -32,7 +32,6 @@ jobs: platform: ios target: template_release tests: false - scons-cache-limit: 1 - name: Save Godot build cache uses: ./.github/actions/godot-cache-save diff --git a/.github/workflows/linux_builds.yml b/.github/workflows/linux_builds.yml index 4479fe989e8..5a0dea07d3f 100644 --- a/.github/workflows/linux_builds.yml +++ b/.github/workflows/linux_builds.yml @@ -34,7 +34,6 @@ jobs: artifact: true # Validate godot-cpp compatibility on one arbitrary editor build. godot-cpp: true - cache-limit: 2 - name: Editor with doubles and GCC sanitizers (target=editor, tests=yes, dev_build=yes, scu_build=yes, precision=double, use_asan=yes, use_ubsan=yes, linker=gold) cache-name: linux-editor-double-sanitizers @@ -47,7 +46,6 @@ jobs: proj-test: true # Skip 2GiB artifact speeding up action. artifact: false - cache-limit: 7 - name: Editor with clang sanitizers (target=editor, tests=yes, dev_build=yes, use_asan=yes, use_ubsan=yes, use_llvm=yes, linker=lld) cache-name: linux-editor-llvm-sanitizers @@ -60,7 +58,6 @@ jobs: artifact: false # Test our oldest supported SCons/Python versions on one arbitrary editor build. legacy-scons: true - cache-limit: 7 - name: Editor with ThreadSanitizer (target=editor, tests=yes, dev_build=yes, use_tsan=yes, use_llvm=yes, linker=lld) cache-name: linux-editor-thread-sanitizer @@ -71,7 +68,6 @@ jobs: build-mono: false # Skip 2GiB artifact speeding up action. artifact: false - cache-limit: 5 - name: Template w/ Mono, release (target=template_release, tests=yes) cache-name: linux-template-mono @@ -81,7 +77,6 @@ jobs: build-mono: false tests: true artifact: true - cache-limit: 1 - name: Template w/ Mono, debug (target=template_debug, tests=yes) cache-name: linux-template-mono-debug @@ -91,7 +86,6 @@ jobs: build-mono: false tests: true artifact: true - cache-limit: 1 - name: Minimal template (target=template_release, tests=yes, everything disabled) cache-name: linux-template-minimal @@ -100,7 +94,6 @@ jobs: bin: ./bin/godot.linuxbsd.template_release.x86_64 tests: true artifact: true - cache-limit: 1 steps: - name: Checkout @@ -165,7 +158,6 @@ jobs: platform: linuxbsd target: ${{ matrix.target }} tests: ${{ matrix.tests }} - scons-cache-limit: ${{ matrix.cache-limit }} - name: Compilation (godot-cpp) uses: ./.github/actions/godot-cpp-build diff --git a/.github/workflows/macos_builds.yml b/.github/workflows/macos_builds.yml index 272639f7758..74cf57b2840 100644 --- a/.github/workflows/macos_builds.yml +++ b/.github/workflows/macos_builds.yml @@ -20,7 +20,6 @@ jobs: target: editor tests: true bin: ./bin/godot.macos.editor.universal - cache-limit: 1 - name: Template (target=template_release, tests=yes) cache-name: macos-template @@ -28,7 +27,6 @@ jobs: tests: true sconsflags: debug_symbols=no bin: ./bin/godot.macos.template_release.universal - cache-limit: 1 steps: - name: Checkout @@ -56,7 +54,6 @@ jobs: platform: macos target: ${{ matrix.target }} tests: ${{ matrix.tests }} - scons-cache-limit: 0 # Only cap on second run to avoid purging unnecessarily - name: Compilation (arm64) uses: ./.github/actions/godot-build @@ -65,7 +62,6 @@ jobs: platform: macos target: ${{ matrix.target }} tests: ${{ matrix.tests }} - scons-cache-limit: ${{ matrix.cache-limit }} - name: Save Godot build cache uses: ./.github/actions/godot-cache-save diff --git a/.github/workflows/web_builds.yml b/.github/workflows/web_builds.yml index 2a3451307d8..3cbdfa324a4 100644 --- a/.github/workflows/web_builds.yml +++ b/.github/workflows/web_builds.yml @@ -62,7 +62,6 @@ jobs: platform: web target: ${{ matrix.target }} tests: ${{ matrix.tests }} - scons-cache-limit: 0.5 - name: Save Godot build cache uses: ./.github/actions/godot-cache-save diff --git a/.github/workflows/windows_builds.yml b/.github/workflows/windows_builds.yml index 6395ffb96e8..535756025a4 100644 --- a/.github/workflows/windows_builds.yml +++ b/.github/workflows/windows_builds.yml @@ -26,7 +26,6 @@ jobs: sconsflags: debug_symbols=no vsproj=yes vsproj_gen_only=no windows_subsystem=console bin: ./bin/godot.windows.editor.x86_64.exe compiler: msvc - cache-limit: 2 - name: Editor w/ clang-cl (target=editor, tests=yes, use_llvm=yes) cache-name: windows-editor-clang @@ -35,7 +34,6 @@ jobs: sconsflags: debug_symbols=no windows_subsystem=console use_llvm=yes bin: ./bin/godot.windows.editor.x86_64.llvm.exe compiler: clang - cache-limit: 1 - name: Template (target=template_release, tests=yes) cache-name: windows-template @@ -44,7 +42,6 @@ jobs: sconsflags: debug_symbols=no bin: ./bin/godot.windows.template_release.x86_64.console.exe compiler: msvc - cache-limit: 2 - name: Template w/ GCC (target=template_release, tests=yes, use_mingw=yes) cache-name: windows-template-gcc @@ -54,7 +51,6 @@ jobs: sconsflags: debug_symbols=no use_mingw=yes bin: ./bin/godot.windows.template_release.x86_64.console.exe compiler: gcc - cache-limit: 1 steps: - name: Checkout @@ -92,7 +88,6 @@ jobs: platform: windows target: ${{ matrix.target }} tests: ${{ matrix.tests }} - scons-cache-limit: ${{ matrix.cache-limit }} - name: Save Godot build cache uses: ./.github/actions/godot-cache-save diff --git a/methods.py b/methods.py index 697c5c815ca..cdb75f2ab0f 100644 --- a/methods.py +++ b/methods.py @@ -897,11 +897,9 @@ def prepare_cache(env) -> None: # Convert GiB to bytes; treat negative numbers as 0 (unlimited). cache_limit = max(0, int(cache_limit * 1024 * 1024 * 1024)) if env["verbose"]: - print( - "Current cache limit is {} (used: {})".format( - convert_size(cache_limit) if cache_limit else "∞", - convert_size(get_size(cache_path)), - ) + print_info( + f"Current cache size is {convert_size(get_size(cache_path))}" + + (f" (limit: {convert_size(cache_limit)})" if cache_limit else "") ) atexit.register(clean_cache, cache_path, cache_limit, env["verbose"]) diff --git a/misc/scripts/purge_cache.py b/misc/scripts/purge_cache.py new file mode 100755 index 00000000000..5f9b584b0d5 --- /dev/null +++ b/misc/scripts/purge_cache.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import argparse +import glob +import os + +if __name__ != "__main__": + raise ImportError(f"{__name__} should not be used as a module.") + + +def main(): + parser = argparse.ArgumentParser(description="Cleanup old cache files") + parser.add_argument("timestamp", type=int, help="Unix timestamp cutoff") + parser.add_argument("directory", help="Path to cache directory") + args = parser.parse_args() + + ret = 0 + + # TODO: Convert to non-hardcoded path + if os.path.exists("redundant.txt"): + with open("redundant.txt") as redundant: + for item in map(str.strip, redundant): + if os.path.isfile(item): + try: + os.remove(item) + except OSError: + print(f'Failed to handle "{item}"; skipping.') + ret += 1 + + for file in glob.glob(os.path.join(args.directory, "*", "*")): + try: + if os.path.getatime(file) < args.timestamp: + os.remove(file) + except OSError: + print(f'Failed to handle "{file}"; skipping.') + ret += 1 + + return ret + + +try: + raise SystemExit(main()) +except KeyboardInterrupt: + import signal + + signal.signal(signal.SIGINT, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGINT)