Merge bitcoin/bitcoin#35430: ci: use warp caching on warp runners

2ce4ae7d8f ci: Add dynamic cache switching to warp cache (will)

Pull request description:

  The GHA cache is very slow, taking on the order of minutes to save and
  restore from.

  Use WarpBuild's cache instead as this is in the same region and should
  be much faster.

  WarpBuild cache action does not auto-fallback to GHA if not being run on
  Warp. To allow fork runs to fallback to GHA caching, whilst minimising
  duplication in the action files, create new "interal" actions which
  perform the switching logic, and use these in the (renamed) cache|save
  actions.

  Without this we would need the `if` logic in our prvious actions, 4
  times in each of save and restore.

ACKs for top commit:
  m3dwards:
    ACK 2ce4ae7d8f

Tree-SHA512: 966ed85558e8ebeed20a6fdf065f0838c9bee95f7d0a2fd1c8a46de20b956f5eedb6bf5098d742da20ea80c56069b86ff46f3151936dc28d5e4afa70acc5d988

.github/actions/cache/restore/action.yml

@@ -1,43 +1,51 @@
 name: 'Restore Caches'
 description: 'Restore ccache, depends sources, and built depends caches'
+inputs:
+  provider:
+    description: 'The cache provider to use'
+    required: true
 runs:
   using: 'composite'
   steps:
     - name: Restore Ccache cache
       id: ccache-cache
-      uses: actions/cache/restore@v5
+      uses: ./.github/actions/cache/restore/internal
       with:
         path: ${{ env.CCACHE_DIR }}
         key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }}
         restore-keys: |
           ccache-${{ env.CONTAINER_NAME }}-
+        provider: ${{ inputs.provider }}
 
     - name: Restore depends sources cache
       id: depends-sources
-      uses: actions/cache/restore@v5
+      uses: ./.github/actions/cache/restore/internal
       with:
         path: ${{ env.SOURCES_PATH }}
         key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
         restore-keys: |
           depends-sources-${{ env.CONTAINER_NAME }}-
+        provider: ${{ inputs.provider }}
 
     - name: Restore built depends cache
       id: depends-built
-      uses: actions/cache/restore@v5
+      uses: ./.github/actions/cache/restore/internal
       with:
         path: ${{ env.BASE_CACHE }}
         key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
         restore-keys: |
           depends-built-${{ env.CONTAINER_NAME }}-
+        provider: ${{ inputs.provider }}
 
     - name: Restore previous releases cache
       id: previous-releases
-      uses: actions/cache/restore@v5
+      uses: ./.github/actions/cache/restore/internal
       with:
         path: ${{ env.PREVIOUS_RELEASES_DIR }}
         key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }}
         restore-keys: |
           previous-releases-${{ env.CONTAINER_NAME }}-
+        provider: ${{ inputs.provider }}
 
     - name: export cache hits
       shell: bash

.github/actions/cache/restore/internal/action.yml

@@ -0,0 +1,43 @@
+name: 'Cache Restore'
+description: 'Restore a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise'
+inputs:
+  path:
+    description: 'A list of files, directories, and wildcard patterns to restore'
+    required: true
+  key:
+    description: 'An explicit key for restoring the cache'
+    required: true
+  restore-keys:
+    description: 'An ordered multiline string listing prefix-matched restore keys'
+    required: false
+    default: ''
+  provider:
+    description: 'The cache provider to use'
+    required: true
+outputs:
+  cache-hit:
+    description: 'A boolean value to indicate an exact match was found for the primary key'
+    value: ${{ steps.warp.outputs.cache-hit || steps.gha.outputs.cache-hit }}
+  cache-primary-key:
+    description: 'The primary key used to restore the cache'
+    value: ${{ steps.warp.outputs.cache-primary-key || steps.gha.outputs.cache-primary-key }}
+runs:
+  using: 'composite'
+  steps:
+    - name: Restore cache with WarpBuild
+      id: warp
+      if: ${{ inputs.provider == 'warp' }}
+      uses: WarpBuilds/cache/restore@v1
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: Restore cache with GitHub Actions
+      id: gha
+      if: ${{ inputs.provider == 'gha' }}
+      uses: actions/cache/restore@v5
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}

.github/actions/cache/save/action.yml

@@ -1,5 +1,9 @@
 name: 'Save Caches'
 description: 'Save ccache, depends sources, and built depends caches'
+inputs:
+  provider:
+    description: 'The cache provider to use'
+    required: true
 runs:
   using: 'composite'
   steps:
@@ -11,29 +15,33 @@ runs:
         echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}"
 
     - name: Save Ccache cache
-      uses: actions/cache/save@v5
+      uses: ./.github/actions/cache/save/internal
       if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }}
       with:
         path: ${{ env.CCACHE_DIR }}
         key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }}
+        provider: ${{ inputs.provider }}
 
     - name: Save depends sources cache
-      uses: actions/cache/save@v5
+      uses: ./.github/actions/cache/save/internal
       if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }}
       with:
         path: ${{ env.SOURCES_PATH }}
         key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
+        provider: ${{ inputs.provider }}
 
     - name: Save built depends cache
-      uses: actions/cache/save@v5
+      uses: ./.github/actions/cache/save/internal
       if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}}
       with:
         path: ${{ env.BASE_CACHE }}
         key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
+        provider: ${{ inputs.provider }}
 
     - name: Save previous releases cache
-      uses: actions/cache/save@v5
+      uses: ./.github/actions/cache/save/internal
       if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}}
       with:
         path: ${{ env.PREVIOUS_RELEASES_DIR }}
         key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }}
+        provider: ${{ inputs.provider }}

.github/actions/cache/save/internal/action.yml

@@ -0,0 +1,28 @@
+name: 'Cache Save'
+description: 'Save a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise'
+inputs:
+  path:
+    description: 'A list of files, directories, and wildcard patterns to cache'
+    required: true
+  key:
+    description: 'An explicit key for saving the cache'
+    required: true
+  provider:
+    description: 'The cache provider to use'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Save cache with WarpBuild
+      if: ${{ inputs.provider == 'warp' }}
+      uses: WarpBuilds/cache/save@v1
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+
+    - name: Save cache with GitHub Actions
+      if: ${{ inputs.provider == 'gha' }}
+      uses: actions/cache/save@v5
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}

.github/workflows/ci.yml

@@ -357,7 +357,9 @@ jobs:
 
       - name: Restore caches
         id: restore-cache
-        uses: ./.github/actions/restore-caches
+        uses: ./.github/actions/cache/restore
+        with:
+          provider: ${{ needs.runners.outputs.provider }}
 
       - name: Configure Docker
         uses: ./.github/actions/configure-docker
@@ -366,7 +368,9 @@ jobs:
         run: ./ci/test_run_all.sh
 
       - name: Save caches
-        uses: ./.github/actions/save-caches
+        uses: ./.github/actions/cache/save
+        with:
+          provider: ${{ needs.runners.outputs.provider }}
 
       - name: Upload built executables
         uses: actions/upload-artifact@v7
@@ -547,7 +551,9 @@ jobs:
 
       - name: Restore caches
         id: restore-cache
-        uses: ./.github/actions/restore-caches
+        uses: ./.github/actions/cache/restore
+        with:
+          provider: ${{ matrix.provider || needs.runners.outputs.provider }}
 
       - name: Configure Docker
         uses: ./.github/actions/configure-docker
@@ -571,7 +577,9 @@ jobs:
         run: ./ci/test_run_all.sh
 
       - name: Save caches
-        uses: ./.github/actions/save-caches
+        uses: ./.github/actions/cache/save
+        with:
+          provider: ${{ matrix.provider || needs.runners.outputs.provider }}
 
   lint:
     name: 'lint'

ci/README.md

@@ -87,8 +87,10 @@ To configure the primary repository, follow these steps:
 4. Permit the following actions to run:
    1. actions/cache/restore@\*
    1. actions/cache/save@\*
-   1. docker/setup-buildx-action@\*
    1. actions/github-script@\*
+   1. docker/setup-buildx-action@\*
+   1. warpbuilds/cache/restore@\*
+   1. warpbuilds/cache/save@\*
 
 ### Forked repositories