From 583b8562abc8472941fb8c03da09eff6952e0b74 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 08:42:55 +0800 Subject: [PATCH 01/10] .github: use module cache only whenever possible --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6046f3efd..2b76c9e4f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,6 +54,7 @@ jobs: uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' + use-build-cache: 'no' - name: docker image cache uses: satackey/action-docker-layer-caching@v0.0.11 @@ -203,6 +204,7 @@ jobs: uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' + use-build-cache: 'no' - name: check default values in sample-lnd.conf file run: make sample-conf-check From 77380debbcdf4d0f0a01a21fcf31eabe8603d4c4 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 08:52:00 +0800 Subject: [PATCH 02/10] .github: replace the aggressive cleanup with a more targeted one This follows GitHub's official recommendation to avoid re-downloading essential tools that setup-go might depend on --- .github/workflows/main.yml | 56 +++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b76c9e4f..b13cb9717 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,7 +45,10 @@ jobs: runs-on: ubuntu-latest steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -72,7 +75,10 @@ jobs: runs-on: ubuntu-latest steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -103,7 +109,10 @@ jobs: runs-on: ubuntu-latest steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -129,7 +138,10 @@ jobs: runs-on: ubuntu-latest steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -172,7 +184,10 @@ jobs: sys: darwin-arm64 freebsd-arm linux-armv6 linux-armv7 linux-arm64 windows-arm steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -195,7 +210,10 @@ jobs: runs-on: ubuntu-latest steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -231,7 +249,10 @@ jobs: steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -300,7 +321,10 @@ jobs: args: backend=neutrino cover=1 steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -379,6 +403,12 @@ jobs: - name: bitcoind-postgres-nativesql-experiment args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql steps: + - name: cleanup space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + - name: git checkout uses: actions/checkout@v4 with: @@ -527,7 +557,10 @@ jobs: steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 @@ -544,7 +577,10 @@ jobs: if: '!contains(github.event.pull_request.labels.*.name, ''no-changelog'')' steps: - name: cleanup space - run: rm -rf /opt/hostedtoolcache + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc - name: git checkout uses: actions/checkout@v4 From da2b3469dd98441a789ecb9ae1ec662594a76b9c Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 09:34:26 +0800 Subject: [PATCH 03/10] .github: move small tasks onto the same machine Thus saving us from the overhead cost, also they can share the same cache file. --- .github/workflows/main.yml | 105 +++++++++++++------------------------ 1 file changed, 37 insertions(+), 68 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b13cb9717..6ff41bf24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,11 +37,8 @@ env: GO_VERSION: 1.23.10 jobs: - ######################## - # SQLC code gen check - ######################## - sqlc-check: - name: Sqlc check + static-checks: + name: Static Checks runs-on: ubuntu-latest steps: - name: cleanup space @@ -50,54 +47,60 @@ jobs: sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 + with: + # Needed for some checks. + fetch-depth: 0 - - name: setup go ${{ env.GO_VERSION }} + - name: Setup Go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' use-build-cache: 'no' - - name: docker image cache + ######################## + # sample configuration check + ######################## + - name: Check default values in sample-lnd.conf file + run: make sample-conf-check + + ######################## + # Check code and RPC format + ######################## + - name: Check code format + run: make fmt-check + + - name: Check go modules tidiness + run: make tidy-module-check + + - name: Lint proto files + run: make protolint + + ######################## + # SQLC code gen check + ######################## + - name: Docker image cache uses: satackey/action-docker-layer-caching@v0.0.11 # Ignore the failure of a step and avoid terminating the job. continue-on-error: true - - name: Generate sql models + - name: Check SQL models run: make sqlc-check - ######################## - # RPC and mobile compilation check - ######################## - rpc-check: - name: RPC and mobile compilation check - runs-on: ubuntu-latest - steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - - name: git checkout - uses: actions/checkout@v4 - - - name: setup go ${{ env.GO_VERSION }} - uses: ./.github/actions/setup-go - with: - go-version: '${{ env.GO_VERSION }}' - - - name: run check + ######################## + # RPC and mobile compilation check + ######################## + - name: Check RPC format run: make rpc-check - - name: run JSON/WASM stub compilation check + - name: Check JSON/WASM stub compilation run: make rpc-js-compile - - name: build mobile RPC bindings + - name: Check mobile RPC bindings run: make mobile-rpc - - name: build mobile specific code + - name: Check mobile specific code run: go build --tags="mobile" ./mobile ######################## @@ -153,15 +156,6 @@ jobs: with: go-version: '${{ env.GO_VERSION }}' - - name: check code format - run: make fmt-check - - - name: check go modules tidiness - run: make tidy-module-check - - - name: lint proto files - run: make protolint - - name: lint run: GOGC=50 make lint @@ -202,31 +196,6 @@ jobs: - name: build release for all architectures run: make release sys="${{ matrix.sys }}" - ######################## - # sample configuration check - ######################## - sample-conf-check: - name: sample configuration check - runs-on: ubuntu-latest - steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - - name: git checkout - uses: actions/checkout@v4 - - - name: setup go ${{ env.GO_VERSION }} - uses: ./.github/actions/setup-go - with: - go-version: '${{ env.GO_VERSION }}' - use-build-cache: 'no' - - - name: check default values in sample-lnd.conf file - run: make sample-conf-check - ######################## # run unit tests ######################## From 4f98a0313a19855e7a8303143aa948af7f49ced6 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 05:46:50 +0300 Subject: [PATCH 04/10] .github: add write permission to delete caches --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ff41bf24..01d7b64e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,6 +11,12 @@ on: branches: - "master" +permissions: + # Required to manage and delete caches. + actions: write + # Default permission for checking out code. + contents: read + concurrency: # Cancel any previous workflows if they are from a PR or push. group: ${{ github.event.pull_request.number || github.ref }} From db0927fb9588410923e154b9f525817d6fd96e56 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 05:48:02 +0300 Subject: [PATCH 05/10] .github: auto clean caches older than 12 hours --- .github/workflows/main.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 01d7b64e5..9ff98ccb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -579,6 +579,46 @@ jobs: - name: 🛡️ backwards compatibility test run: make backwards-compat-test + ######################################### + # Auto Cache Cleanup on Pull Requests + ######################################### + auto-cleanup-cache: + name: Cache Cleanup + runs-on: ubuntu-latest + + # This condition checks for pull requests from authors with write access. + if: >- + contains('OWNER, MEMBER, COLLABORATOR', github.event.pull_request.author_association) + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Delete caches older than 12 hours + continue-on-error: true + env: + # GITHUB_TOKEN is required for the gh CLI. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Finding caches not used in the last 12 hours..." + + # Get the current time and the cutoff time (12 hours ago) in Unix + # timestamp format. + cutoff_timestamp=$(date -d "12 hours ago" +%s) + + # Use gh and jq to parse caches. Delete any cache last accessed + # before the cutoff time. + gh cache list --json key,lastAccessedAt | jq -r '.[] | + select(.lastAccessedAt != null) | "\(.lastAccessedAt) \(.key)"' | + while read -r last_accessed_at key; do + last_accessed_timestamp=$(date -d "$last_accessed_at" +%s) + + if (( last_accessed_timestamp < cutoff_timestamp )); then + echo "Deleting old cache. Key: $key, Last Used: $last_accessed_at" + gh cache delete "$key" + fi + done + # Notify about the completion of all coverage collecting jobs. finish: if: ${{ always() }} From 81ce5aad9437b4c564a4f80d3301ba32c68b5acd Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 06:36:22 +0300 Subject: [PATCH 06/10] .github: run PR review stats only for side branches --- .github/workflows/stats.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index b3969775a..62b79f9b0 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -4,12 +4,21 @@ on: pull_request: types: [opened] +permissions: + # Required to post stats as comments. + actions: write + # Default permission for checking out code. + contents: read + jobs: stats: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write + + # Check if the PR is from the base repo (not a fork). Only the + # collaborators have the permission to create a side branch from the base + # repo, so this implicitly restricts who can run this job. + if: github.event.pull_request.head.repo.fork == false + steps: - name: Run pull request stats uses: flowwer-dev/pull-request-stats@v2.11.0 From a83dcaffae42b13b722c5e05e48139ef470d9360 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 14:54:05 +0300 Subject: [PATCH 07/10] .github: add new action `cleanup-space` --- .github/actions/cleanup-space/action.yml | 16 +++++ .github/workflows/main.yml | 81 ++++++++---------------- 2 files changed, 43 insertions(+), 54 deletions(-) create mode 100644 .github/actions/cleanup-space/action.yml diff --git a/.github/actions/cleanup-space/action.yml b/.github/actions/cleanup-space/action.yml new file mode 100644 index 000000000..037435f98 --- /dev/null +++ b/.github/actions/cleanup-space/action.yml @@ -0,0 +1,16 @@ +name: "Clean up runner disk space" +description: "Removes large, non-essential toolsets to free up disk space on the runner." + +runs: + using: "composite" + steps: + - name: Free up disk space + shell: bash + run: | + echo "Removing large toolsets to free up disk space..." + # Remove dotnet to save disk space. + sudo rm -rf /usr/share/dotnet + # Remove android to save disk space. + sudo rm -rf /usr/local/lib/android + # Remove ghc to save disk space. + sudo rm -rf /opt/ghc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ff98ccb0..d1fa4b52a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,18 +47,15 @@ jobs: name: Static Checks runs-on: ubuntu-latest steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: Git checkout uses: actions/checkout@v4 with: # Needed for some checks. fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: Setup Go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: @@ -117,17 +114,14 @@ jobs: name: check commits runs-on: ubuntu-latest steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: @@ -146,17 +140,14 @@ jobs: name: lint code runs-on: ubuntu-latest steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: @@ -183,15 +174,12 @@ jobs: - name: arm sys: darwin-arm64 freebsd-arm linux-armv6 linux-armv7 linux-arm64 windows-arm steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: @@ -223,17 +211,14 @@ jobs: - unit-module steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase @@ -295,17 +280,14 @@ jobs: - name: neutrino args: backend=neutrino cover=1 steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase @@ -378,17 +360,14 @@ jobs: - name: bitcoind-postgres-nativesql-experiment args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase @@ -531,15 +510,12 @@ jobs: - github.com/golang/protobuf v1.5.3 steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: ensure dependencies at correct version run: if ! grep -q "${{ matrix.pinned_dep }}" go.mod; then echo dependency ${{ matrix.pinned_dep }} should not be altered ; exit 1 ; fi @@ -551,15 +527,12 @@ jobs: runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-changelog'')' steps: - - name: cleanup space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - - name: git checkout uses: actions/checkout@v4 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: release notes check run: scripts/check-release-notes.sh From 4b1a68778ea6d107c660d7e38a3c80103b5ac236 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Fri, 4 Jul 2025 15:47:09 +0300 Subject: [PATCH 08/10] .github: use title case in job names To unify the format used in job names. --- .github/workflows/main.yml | 95 +++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1fa4b52a..ded83acbe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: ######################## check-commits: if: github.event_name == 'pull_request' - name: check commits + name: Check commits runs-on: ubuntu-latest steps: - name: git checkout @@ -137,7 +137,7 @@ jobs: # lint code ######################## lint: - name: lint code + name: Lint code runs-on: ubuntu-latest steps: - name: git checkout @@ -160,7 +160,7 @@ jobs: # cross compilation ######################## cross-compile: - name: cross compilation + name: Cross compilation runs-on: ubuntu-latest strategy: fail-fast: true @@ -174,27 +174,27 @@ jobs: - name: arm sys: darwin-arm64 freebsd-arm linux-armv6 linux-armv7 linux-arm64 windows-arm steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - name: Clean up runner space uses: ./.github/actions/cleanup-space - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: cross-compile use-build-cache: 'no' - - name: build release for all architectures + - name: Build release for all architectures run: make release sys="${{ matrix.sys }}" ######################## # run unit tests ######################## unit-test: - name: run unit tests + name: Run unit tests runs-on: ubuntu-latest strategy: # Allow other tests in the matrix to continue if one fails. @@ -211,7 +211,7 @@ jobs: - unit-module steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 @@ -219,29 +219,29 @@ jobs: - name: Clean up runner space uses: ./.github/actions/cleanup-space - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: git checkout fuzzing seeds + - name: Git checkout fuzzing seeds uses: actions/checkout@v4 with: repository: lightninglabs/lnd-fuzz path: lnd-fuzz - - name: rsync fuzzing seeds + - name: Rsync fuzzing seeds run: rsync -a --ignore-existing lnd-fuzz/ ./ - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: unit-test - - name: install bitcoind + - name: Install bitcoind run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION - - name: run ${{ matrix.unit_type }} + - name: Run ${{ matrix.unit_type }} run: make ${{ matrix.unit_type }} - name: Clean coverage @@ -263,7 +263,7 @@ jobs: # run integration tests with TRANCHES ######################## basic-integration-test: - name: basic itests + name: Run basic itests runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' strategy: @@ -280,7 +280,7 @@ jobs: - name: neutrino args: backend=neutrino cover=1 steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 @@ -288,20 +288,20 @@ jobs: - name: Clean up runner space uses: ./.github/actions/cleanup-space - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: install bitcoind + - name: Install bitcoind run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION - - name: run ${{ matrix.name }} + - name: Run ${{ matrix.name }} run: make itest-parallel tranches=${{ env.TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }} - name: Clean coverage @@ -335,7 +335,7 @@ jobs: # run integration tests with SMALL_TRANCHES ######################## integration-test: - name: itests + name: Run itests runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' strategy: @@ -360,7 +360,7 @@ jobs: - name: bitcoind-postgres-nativesql-experiment args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 @@ -368,20 +368,20 @@ jobs: - name: Clean up runner space uses: ./.github/actions/cleanup-space - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: install bitcoind + - name: Install bitcoind run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION - - name: run ${{ matrix.name }} + - name: Run ${{ matrix.name }} run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }} - name: Clean coverage @@ -416,29 +416,29 @@ jobs: # run windows integration test ######################## windows-integration-test: - name: windows itest + name: Run windows itest runs-on: windows-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: run itest + - name: Run itest run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} windows=1 shuffleseed=${{ github.run_id }} - - name: kill any remaining lnd processes + - name: Kill any remaining lnd processes if: ${{ failure() }} shell: powershell run: taskkill /IM lnd-itest.exe /T /F @@ -460,26 +460,26 @@ jobs: # run macOS integration test ######################## macos-integration-test: - name: macOS itest + name: Run macOS itest runs-on: macos-14 if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: run itest + - name: Run itest run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} shuffleseed=${{ github.run_id }} - name: Zip log files on failure @@ -499,7 +499,7 @@ jobs: # check pinned dependencies ######################## dep-pin: - name: check pinned dependencies + name: Check pinned dependencies runs-on: ubuntu-latest strategy: # Allow other tests in the matrix to continue if one fails. @@ -510,46 +510,46 @@ jobs: - github.com/golang/protobuf v1.5.3 steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - name: Clean up runner space uses: ./.github/actions/cleanup-space - - name: ensure dependencies at correct version + - name: Ensure dependencies at correct version run: if ! grep -q "${{ matrix.pinned_dep }}" go.mod; then echo dependency ${{ matrix.pinned_dep }} should not be altered ; exit 1 ; fi ######################## # check PR updates release notes ######################## milestone-check: - name: check release notes updated + name: Check release notes updated runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-changelog'')' steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - name: Clean up runner space uses: ./.github/actions/cleanup-space - - name: release notes check + - name: Release notes check run: scripts/check-release-notes.sh ######################## # Backwards Compatibility Test ######################## - backwards-compatability-test: - name: backwards compatability test + backwards-compatibility-test: + name: Backwards compatibility test runs-on: ubuntu-latest steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - name: 🐳 Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: 🛡️ backwards compatibility test + - name: 🛡️ Backwards compatibility test run: make backwards-compat-test ######################################### @@ -594,6 +594,7 @@ jobs: # Notify about the completion of all coverage collecting jobs. finish: + name: Send coverage report if: ${{ always() }} needs: [unit-test, basic-integration-test] runs-on: ubuntu-latest From f2e1c11593772770f4a8b9d5b610a150b449f938 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Thu, 17 Jul 2025 16:31:36 +0800 Subject: [PATCH 09/10] .github: remove `github.job` from the cache key and update docs This commit adds a more descriptive docs about the caching process. In addition, the `github.job` is now removed from the cache key construction to allow jobs sharing the same cache. In addition, given we now use `inputs.key-prefix`, the `github.job` is no longer relevant as the `key-prefix` already creates a unique name for the cache. We also update the lint and check commits job to use the same cache from the unit test, given they are running in the same environment. --- .github/actions/setup-go/action.yml | 37 +++++++++++++++++++++++++---- .github/workflows/main.yml | 4 ++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index b6d85c030..aa6a50f51 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -24,6 +24,16 @@ runs: go-version: '${{ inputs.go-version }}' cache: 'false' + # When we run go build or go test, the Go compiler calculates a signature + # for each package based on the content of its `.go` source files, its + # dependencies, and the compiler flags. It then checks the restored build + # cache for an entry matching that exact signature to speed up the jobs. + # - Cache Hit: If an entry exists (meaning the source files and + # dependencies haven't changed), it reuses the compiled artifact directly + # from the cache. + # - Cache Miss: If no entry exists (because we changed a line of code), it + # recompiles that specific package and stores the new result in the cache + # for the next time. - name: go cache if: ${{ inputs.use-build-cache == 'yes' }} uses: actions/cache@v4 @@ -38,11 +48,31 @@ runs: ~/.cache/go-build ~/Library/Caches/go-build ~\AppData\Local\go-build - key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}-${{ hashFiles('**/go.sum') }} + + # The key is used to create and later look up the cache. It's made of + # four parts: + # - The base part is made from the OS name, Go version and a + # job-specified key prefix. Example: `linux-go-1.23.10-unit-test-`. + # It ensures that a job running on Linux with Go 1.23 only looks for + # caches from the same environment. + # - The unique part is the `hashFiles('**/go.sum')`, which calculates a + # hash (a fingerprint) of the go.sum file. + key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ hashFiles('**/go.sum') }} + + # The restore-keys provides a list of fallback keys. If no cache + # matches the key exactly, the action will look for a cache where the + # key starts with one of the restore-keys. The action searches the + # restore-keys list in order and restores the most recently created + # cache that matches the prefix. Once the job is done, a new cache is + # created and saved using the new key. restore-keys: | - ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}- ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}- + # The complete, downloaded source code of all our dependencies (the + # libraries lnd project imports). This prevents the go command from having + # to re-download every dependency from the internet on every single job + # run. It's like having a local library of all the third-party code lnd + # needs. - name: go module cache if: ${{ inputs.use-build-cache == 'no' }} uses: actions/cache@v4 @@ -50,9 +80,8 @@ runs: # Just the module download cache. path: | ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}- ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache- - name: set GOPATH diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ded83acbe..a43c2cf54 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -126,6 +126,8 @@ jobs: uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' + # Use the same cache from unit test job to save time. + key-prefix: unit-test - name: fetch and rebase on ${{ github.base_ref }} uses: ./.github/actions/rebase @@ -152,6 +154,8 @@ jobs: uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' + # Use the same cache from unit test job to save time. + key-prefix: unit-test - name: lint run: GOGC=50 make lint From af2f935ab72c4188ece67d3d1766a9ebc63a9915 Mon Sep 17 00:00:00 2001 From: yyforyongyu Date: Thu, 17 Jul 2025 21:00:35 +0800 Subject: [PATCH 10/10] .github: add timeout when sending coverage reports We also make it cancelable. --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a43c2cf54..34675e827 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -599,9 +599,10 @@ jobs: # Notify about the completion of all coverage collecting jobs. finish: name: Send coverage report - if: ${{ always() }} + if: ${{ !cancelled() }} needs: [unit-test, basic-integration-test] runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Send coverage uses: coverallsapp/github-action@v2