From 2ce4ae7d8f006959cb83be93a17b8125e953c30a Mon Sep 17 00:00:00 2001 From: will Date: Mon, 1 Jun 2026 10:26:19 +0100 Subject: [PATCH] ci: Add dynamic cache switching to warp cache The GHA cache is very slow, taking on the order of minutes to save and restore from. Use WarpBuild's cache instead as this is in the same region and much faster. WarpBuild cache action does not auto-fallback to GHA if not being run on Warp. To allow fork runs to fallback to GHA caching, whilst minimising duplication in the action files, create new "interal" actions which perform the switching logic, and use these in the (renamed) cache|save actions. Without this we would need the `if` logic in our prvious actions, 4 times in each of save and restore. Plumb the provider through into the action, as a composite action can't read `env` (`GITHUB_OUTPUT`) from previous steps. --- .../restore}/action.yml | 16 +++++-- .../actions/cache/restore/internal/action.yml | 43 +++++++++++++++++++ .../{save-caches => cache/save}/action.yml | 16 +++++-- .../actions/cache/save/internal/action.yml | 28 ++++++++++++ .github/workflows/ci.yml | 16 +++++-- ci/README.md | 4 +- 6 files changed, 110 insertions(+), 13 deletions(-) rename .github/actions/{restore-caches => cache/restore}/action.yml (77%) create mode 100644 .github/actions/cache/restore/internal/action.yml rename .github/actions/{save-caches => cache/save}/action.yml (79%) create mode 100644 .github/actions/cache/save/internal/action.yml diff --git a/.github/actions/restore-caches/action.yml b/.github/actions/cache/restore/action.yml similarity index 77% rename from .github/actions/restore-caches/action.yml rename to .github/actions/cache/restore/action.yml index ee07d68b907..2cc5b53a1e1 100644 --- a/.github/actions/restore-caches/action.yml +++ b/.github/actions/cache/restore/action.yml @@ -1,43 +1,51 @@ name: 'Restore Caches' description: 'Restore ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: - name: Restore Ccache cache id: ccache-cache - uses: actions/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} restore-keys: | ccache-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore depends sources cache id: depends-sources - uses: actions/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-sources-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore built depends cache id: depends-built - uses: actions/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-built-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore previous releases cache id: previous-releases - uses: actions/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} restore-keys: | previous-releases-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: export cache hits shell: bash diff --git a/.github/actions/cache/restore/internal/action.yml b/.github/actions/cache/restore/internal/action.yml new file mode 100644 index 00000000000..43dd206345d --- /dev/null +++ b/.github/actions/cache/restore/internal/action.yml @@ -0,0 +1,43 @@ +name: 'Cache Restore' +description: 'Restore a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to restore' + required: true + key: + description: 'An explicit key for restoring the cache' + required: true + restore-keys: + description: 'An ordered multiline string listing prefix-matched restore keys' + required: false + default: '' + provider: + description: 'The cache provider to use' + required: true +outputs: + cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' + value: ${{ steps.warp.outputs.cache-hit || steps.gha.outputs.cache-hit }} + cache-primary-key: + description: 'The primary key used to restore the cache' + value: ${{ steps.warp.outputs.cache-primary-key || steps.gha.outputs.cache-primary-key }} +runs: + using: 'composite' + steps: + - name: Restore cache with WarpBuild + id: warp + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/restore@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} + + - name: Restore cache with GitHub Actions + id: gha + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} diff --git a/.github/actions/save-caches/action.yml b/.github/actions/cache/save/action.yml similarity index 79% rename from .github/actions/save-caches/action.yml rename to .github/actions/cache/save/action.yml index 8667c91473d..5c543b3f9e7 100644 --- a/.github/actions/save-caches/action.yml +++ b/.github/actions/cache/save/action.yml @@ -1,5 +1,9 @@ name: 'Save Caches' description: 'Save ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: @@ -11,29 +15,33 @@ runs: echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}" - name: Save Ccache cache - uses: actions/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }} with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} + provider: ${{ inputs.provider }} - name: Save depends sources cache - uses: actions/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }} with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save built depends cache - uses: actions/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}} with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save previous releases cache - uses: actions/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}} with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} + provider: ${{ inputs.provider }} diff --git a/.github/actions/cache/save/internal/action.yml b/.github/actions/cache/save/internal/action.yml new file mode 100644 index 00000000000..12f4156a072 --- /dev/null +++ b/.github/actions/cache/save/internal/action.yml @@ -0,0 +1,28 @@ +name: 'Cache Save' +description: 'Save a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to cache' + required: true + key: + description: 'An explicit key for saving the cache' + required: true + provider: + description: 'The cache provider to use' + required: true +runs: + using: 'composite' + steps: + - name: Save cache with WarpBuild + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/save@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + + - name: Save cache with GitHub Actions + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/save@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c44d67801c..8eaa68ab0e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,7 +357,9 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ needs.runners.outputs.provider }} - name: Configure Docker uses: ./.github/actions/configure-docker @@ -366,7 +368,9 @@ jobs: run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ needs.runners.outputs.provider }} - name: Upload built executables uses: actions/upload-artifact@v7 @@ -547,7 +551,9 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Configure Docker uses: ./.github/actions/configure-docker @@ -571,7 +577,9 @@ jobs: run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ matrix.provider || needs.runners.outputs.provider }} lint: name: 'lint' diff --git a/ci/README.md b/ci/README.md index 314d336b76c..0b6a34460ca 100644 --- a/ci/README.md +++ b/ci/README.md @@ -87,8 +87,10 @@ To configure the primary repository, follow these steps: 4. Permit the following actions to run: 1. actions/cache/restore@\* 1. actions/cache/save@\* - 1. docker/setup-buildx-action@\* 1. actions/github-script@\* + 1. docker/setup-buildx-action@\* + 1. warpbuilds/cache/restore@\* + 1. warpbuilds/cache/save@\* ### Forked repositories