highperfocused/lnd
1
0
Fork 0
mirror of https://github.com/lightningnetwork/lnd.git synced 2025-08-25 05:02:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10032 from yyforyongyu/fix-ci-cache

Browse Source 
CI: reduce cache size and auto clean caches
This commit is contained in:
Oliver Gugger
2025-07-18 03:07:34 -06:00
committed by GitHub
parent 47e9f05ddf af2f935ab7
commit 5c62e904e2
4 changed files with 226 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/actions/cleanup-space/action.yml vendored Normal file
View File

@@ -0,0 +1,16 @@
name: "Clean up runner disk space"
description: "Removes large, non-essential toolsets to free up disk space on the runner."
runs:
using: "composite"
steps:
- name: Free up disk space
shell: bash
run: |
echo "Removing large toolsets to free up disk space..."
# Remove dotnet to save disk space.
sudo rm -rf /usr/share/dotnet
# Remove android to save disk space.
sudo rm -rf /usr/local/lib/android
# Remove ghc to save disk space.
sudo rm -rf /opt/ghc

37
.github/actions/setup-go/action.yml vendored
View File

@@ -24,6 +24,16 @@ runs:
go-version: '${{ inputs.go-version }}'
cache: 'false'
# When we run go build or go test, the Go compiler calculates a signature
# for each package based on the content of its `.go` source files, its
# dependencies, and the compiler flags. It then checks the restored build
# cache for an entry matching that exact signature to speed up the jobs.
# - Cache Hit: If an entry exists (meaning the source files and
# dependencies haven't changed), it reuses the compiled artifact directly
# from the cache.
# - Cache Miss: If no entry exists (because we changed a line of code), it
# recompiles that specific package and stores the new result in the cache
# for the next time.
- name: go cache
if: ${{ inputs.use-build-cache == 'yes' }}
uses: actions/cache@v4
@@ -38,11 +48,31 @@ runs:
~/.cache/go-build
~/Library/Caches/go-build
~\AppData\Local\go-build
key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
# The key is used to create and later look up the cache. It's made of
# four parts:
# - The base part is made from the OS name, Go version and a
# job-specified key prefix. Example: `linux-go-1.23.10-unit-test-`.
# It ensures that a job running on Linux with Go 1.23 only looks for
# caches from the same environment.
# - The unique part is the `hashFiles('**/go.sum')`, which calculates a
# hash (a fingerprint) of the go.sum file.
key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ hashFiles('**/go.sum') }}
# The restore-keys provides a list of fallback keys. If no cache
# matches the key exactly, the action will look for a cache where the
# key starts with one of the restore-keys. The action searches the
# restore-keys list in order and restores the most recently created
# cache that matches the prefix. Once the job is done, a new cache is
# created and saved using the new key.
restore-keys: |
${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}-
${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-
# The complete, downloaded source code of all our dependencies (the
# libraries lnd project imports). This prevents the go command from having
# to re-download every dependency from the internet on every single job
# run. It's like having a local library of all the third-party code lnd
# needs.
- name: go module cache
if: ${{ inputs.use-build-cache == 'no' }}
uses: actions/cache@v4
@@ -50,9 +80,8 @@ runs:
# Just the module download cache.
path: |
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}-
${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-
- name: set GOPATH

298
.github/workflows/main.yml vendored
View File

@@ -11,6 +11,12 @@ on:
branches:
- "master"
permissions:
# Required to manage and delete caches.
actions: write
# Default permission for checking out code.
contents: read
concurrency:
# Cancel any previous workflows if they are from a PR or push.
group: ${{ github.event.pull_request.number || github.ref }}
@@ -37,60 +43,67 @@ env:
GO_VERSION: 1.23.10
jobs:
########################
# SQLC code gen check
########################
sqlc-check:
name: Sqlc check
static-checks:
name: Static Checks
runs-on: ubuntu-latest
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
with:
# Needed for some checks.
fetch-depth: 0
- name: setup go ${{ env.GO_VERSION }}
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Setup Go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
use-build-cache: 'no'
- name: docker image cache
########################
# sample configuration check
########################
- name: Check default values in sample-lnd.conf file
run: make sample-conf-check
########################
# Check code and RPC format
########################
- name: Check code format
run: make fmt-check
- name: Check go modules tidiness
run: make tidy-module-check
- name: Lint proto files
run: make protolint
########################
# SQLC code gen check
########################
- name: Docker image cache
uses: satackey/action-docker-layer-caching@v0.0.11
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Generate sql models
- name: Check SQL models
run: make sqlc-check
########################
# RPC and mobile compilation check
########################
rpc-check:
name: RPC and mobile compilation check
runs-on: ubuntu-latest
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
uses: actions/checkout@v4
- name: setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
- name: run check
########################
# RPC and mobile compilation check
########################
- name: Check RPC format
run: make rpc-check
- name: run JSON/WASM stub compilation check
- name: Check JSON/WASM stub compilation
run: make rpc-js-compile
- name: build mobile RPC bindings
- name: Check mobile RPC bindings
run: make mobile-rpc
- name: build mobile specific code
- name: Check mobile specific code
run: go build --tags="mobile" ./mobile
########################
@@ -98,21 +111,23 @@ jobs:
########################
check-commits:
if: github.event_name == 'pull_request'
name: check commits
name: Check commits
runs-on: ubuntu-latest
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
# Use the same cache from unit test job to save time.
key-prefix: unit-test
- name: fetch and rebase on ${{ github.base_ref }}
uses: ./.github/actions/rebase
@@ -124,30 +139,23 @@ jobs:
# lint code
########################
lint:
name: lint code
name: Lint code
runs-on: ubuntu-latest
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
- name: check code format
run: make fmt-check
- name: check go modules tidiness
run: make tidy-module-check
- name: lint proto files
run: make protolint
# Use the same cache from unit test job to save time.
key-prefix: unit-test
- name: lint
run: GOGC=50 make lint
@@ -156,7 +164,7 @@ jobs:
# cross compilation
########################
cross-compile:
name: cross compilation
name: Cross compilation
runs-on: ubuntu-latest
strategy:
fail-fast: true
@@ -170,48 +178,27 @@ jobs:
- name: arm
sys: darwin-arm64 freebsd-arm linux-armv6 linux-armv7 linux-arm64 windows-arm
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
- name: setup go ${{ env.GO_VERSION }}
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: cross-compile
use-build-cache: 'no'
- name: build release for all architectures
- name: Build release for all architectures
run: make release sys="${{ matrix.sys }}"
########################
# sample configuration check
########################
sample-conf-check:
name: sample configuration check
runs-on: ubuntu-latest
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
uses: actions/checkout@v4
- name: setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
- name: check default values in sample-lnd.conf file
run: make sample-conf-check
########################
# run unit tests
########################
unit-test:
name: run unit tests
name: Run unit tests
runs-on: ubuntu-latest
strategy:
# Allow other tests in the matrix to continue if one fails.
@@ -228,37 +215,37 @@ jobs:
- unit-module
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: fetch and rebase on ${{ github.base_ref }}
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Fetch and rebase on ${{ github.base_ref }}
if: github.event_name == 'pull_request'
uses: ./.github/actions/rebase
- name: git checkout fuzzing seeds
- name: Git checkout fuzzing seeds
uses: actions/checkout@v4
with:
repository: lightninglabs/lnd-fuzz
path: lnd-fuzz
- name: rsync fuzzing seeds
- name: Rsync fuzzing seeds
run: rsync -a --ignore-existing lnd-fuzz/ ./
- name: setup go ${{ env.GO_VERSION }}
- name: Setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: unit-test
- name: install bitcoind
- name: Install bitcoind
run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION
- name: run ${{ matrix.unit_type }}
- name: Run ${{ matrix.unit_type }}
run: make ${{ matrix.unit_type }}
- name: Clean coverage
@@ -280,7 +267,7 @@ jobs:
# run integration tests with TRANCHES
########################
basic-integration-test:
name: basic itests
name: Run basic itests
runs-on: ubuntu-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
strategy:
@@ -297,28 +284,28 @@ jobs:
- name: neutrino
args: backend=neutrino cover=1
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: fetch and rebase on ${{ github.base_ref }}
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Fetch and rebase on ${{ github.base_ref }}
if: github.event_name == 'pull_request'
uses: ./.github/actions/rebase
- name: setup go ${{ env.GO_VERSION }}
- name: Setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: integration-test
- name: install bitcoind
- name: Install bitcoind
run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION
- name: run ${{ matrix.name }}
- name: Run ${{ matrix.name }}
run: make itest-parallel tranches=${{ env.TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }}
- name: Clean coverage
@@ -352,7 +339,7 @@ jobs:
# run integration tests with SMALL_TRANCHES
########################
integration-test:
name: itests
name: Run itests
runs-on: ubuntu-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
strategy:
@@ -377,25 +364,28 @@ jobs:
- name: bitcoind-postgres-nativesql-experiment
args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql
steps:
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: fetch and rebase on ${{ github.base_ref }}
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Fetch and rebase on ${{ github.base_ref }}
if: github.event_name == 'pull_request'
uses: ./.github/actions/rebase
- name: setup go ${{ env.GO_VERSION }}
- name: Setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: integration-test
- name: install bitcoind
- name: Install bitcoind
run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION
- name: run ${{ matrix.name }}
- name: Run ${{ matrix.name }}
run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }}
- name: Clean coverage
@@ -430,29 +420,29 @@ jobs:
# run windows integration test
########################
windows-integration-test:
name: windows itest
name: Run windows itest
runs-on: windows-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
steps:
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: fetch and rebase on ${{ github.base_ref }}
- name: Fetch and rebase on ${{ github.base_ref }}
if: github.event_name == 'pull_request'
uses: ./.github/actions/rebase
- name: setup go ${{ env.GO_VERSION }}
- name: Setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: integration-test
- name: run itest
- name: Run itest
run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} windows=1 shuffleseed=${{ github.run_id }}
- name: kill any remaining lnd processes
- name: Kill any remaining lnd processes
if: ${{ failure() }}
shell: powershell
run: taskkill /IM lnd-itest.exe /T /F
@@ -474,26 +464,26 @@ jobs:
# run macOS integration test
########################
macos-integration-test:
name: macOS itest
name: Run macOS itest
runs-on: macos-14
if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')'
steps:
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: fetch and rebase on ${{ github.base_ref }}
- name: Fetch and rebase on ${{ github.base_ref }}
if: github.event_name == 'pull_request'
uses: ./.github/actions/rebase
- name: setup go ${{ env.GO_VERSION }}
- name: Setup go ${{ env.GO_VERSION }}
uses: ./.github/actions/setup-go
with:
go-version: '${{ env.GO_VERSION }}'
key-prefix: integration-test
- name: run itest
- name: Run itest
run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} shuffleseed=${{ github.run_id }}
- name: Zip log files on failure
@@ -513,7 +503,7 @@ jobs:
# check pinned dependencies
########################
dep-pin:
name: check pinned dependencies
name: Check pinned dependencies
runs-on: ubuntu-latest
strategy:
# Allow other tests in the matrix to continue if one fails.
@@ -524,53 +514,95 @@ jobs:
- github.com/golang/protobuf v1.5.3
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
- name: ensure dependencies at correct version
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Ensure dependencies at correct version
run: if ! grep -q "${{ matrix.pinned_dep }}" go.mod; then echo dependency ${{ matrix.pinned_dep }} should not be altered ; exit 1 ; fi
########################
# check PR updates release notes
########################
milestone-check:
name: check release notes updated
name: Check release notes updated
runs-on: ubuntu-latest
if: '!contains(github.event.pull_request.labels.*.name, ''no-changelog'')'
steps:
- name: cleanup space
run: rm -rf /opt/hostedtoolcache
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
- name: release notes check
- name: Clean up runner space
uses: ./.github/actions/cleanup-space
- name: Release notes check
run: scripts/check-release-notes.sh
########################
# Backwards Compatibility Test
########################
backwards-compatability-test:
name: backwards compatability test
backwards-compatibility-test:
name: Backwards compatibility test
runs-on: ubuntu-latest
steps:
- name: git checkout
- name: Git checkout
uses: actions/checkout@v4
- name: 🐳 Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: 🛡️ backwards compatibility test
- name: 🛡️ Backwards compatibility test
run: make backwards-compat-test
#########################################
# Auto Cache Cleanup on Pull Requests
#########################################
auto-cleanup-cache:
name: Cache Cleanup
runs-on: ubuntu-latest
# This condition checks for pull requests from authors with write access.
if: >-
contains('OWNER, MEMBER, COLLABORATOR', github.event.pull_request.author_association)
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Delete caches older than 12 hours
continue-on-error: true
env:
# GITHUB_TOKEN is required for the gh CLI.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "Finding caches not used in the last 12 hours..."
# Get the current time and the cutoff time (12 hours ago) in Unix
# timestamp format.
cutoff_timestamp=$(date -d "12 hours ago" +%s)
# Use gh and jq to parse caches. Delete any cache last accessed
# before the cutoff time.
gh cache list --json key,lastAccessedAt | jq -r '.[] |
select(.lastAccessedAt != null) | "\(.lastAccessedAt) \(.key)"' |
while read -r last_accessed_at key; do
last_accessed_timestamp=$(date -d "$last_accessed_at" +%s)
if (( last_accessed_timestamp < cutoff_timestamp )); then
echo "Deleting old cache. Key: $key, Last Used: $last_accessed_at"
gh cache delete "$key"
fi
done
# Notify about the completion of all coverage collecting jobs.
finish:
if: ${{ always() }}
name: Send coverage report
if: ${{ !cancelled() }}
needs: [unit-test, basic-integration-test]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Send coverage
uses: coverallsapp/github-action@v2

15
.github/workflows/stats.yml vendored
View File

@@ -4,12 +4,21 @@ on:
pull_request:
types: [opened]
permissions:
# Required to post stats as comments.
actions: write
# Default permission for checking out code.
contents: read
jobs:
stats:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
# Check if the PR is from the base repo (not a fork). Only the
# collaborators have the permission to create a side branch from the base
# repo, so this implicitly restricts who can run this job.
if: github.event.pull_request.head.repo.fork == false
steps:
- name: Run pull request stats
uses: flowwer-dev/pull-request-stats@v2.11.0