diff --git a/.github/actions/cleanup-space/action.yml b/.github/actions/cleanup-space/action.yml new file mode 100644 index 000000000..037435f98 --- /dev/null +++ b/.github/actions/cleanup-space/action.yml @@ -0,0 +1,16 @@ +name: "Clean up runner disk space" +description: "Removes large, non-essential toolsets to free up disk space on the runner." + +runs: + using: "composite" + steps: + - name: Free up disk space + shell: bash + run: | + echo "Removing large toolsets to free up disk space..." + # Remove dotnet to save disk space. + sudo rm -rf /usr/share/dotnet + # Remove android to save disk space. + sudo rm -rf /usr/local/lib/android + # Remove ghc to save disk space. + sudo rm -rf /opt/ghc diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index b6d85c030..aa6a50f51 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -24,6 +24,16 @@ runs: go-version: '${{ inputs.go-version }}' cache: 'false' + # When we run go build or go test, the Go compiler calculates a signature + # for each package based on the content of its `.go` source files, its + # dependencies, and the compiler flags. It then checks the restored build + # cache for an entry matching that exact signature to speed up the jobs. + # - Cache Hit: If an entry exists (meaning the source files and + # dependencies haven't changed), it reuses the compiled artifact directly + # from the cache. + # - Cache Miss: If no entry exists (because we changed a line of code), it + # recompiles that specific package and stores the new result in the cache + # for the next time. - name: go cache if: ${{ inputs.use-build-cache == 'yes' }} uses: actions/cache@v4 @@ -38,11 +48,31 @@ runs: ~/.cache/go-build ~/Library/Caches/go-build ~\AppData\Local\go-build - key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}-${{ hashFiles('**/go.sum') }} + + # The key is used to create and later look up the cache. It's made of + # four parts: + # - The base part is made from the OS name, Go version and a + # job-specified key prefix. Example: `linux-go-1.23.10-unit-test-`. + # It ensures that a job running on Linux with Go 1.23 only looks for + # caches from the same environment. + # - The unique part is the `hashFiles('**/go.sum')`, which calculates a + # hash (a fingerprint) of the go.sum file. + key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ hashFiles('**/go.sum') }} + + # The restore-keys provides a list of fallback keys. If no cache + # matches the key exactly, the action will look for a cache where the + # key starts with one of the restore-keys. The action searches the + # restore-keys list in order and restores the most recently created + # cache that matches the prefix. Once the job is done, a new cache is + # created and saved using the new key. restore-keys: | - ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-${{ github.job }}- ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}- + # The complete, downloaded source code of all our dependencies (the + # libraries lnd project imports). This prevents the go command from having + # to re-download every dependency from the internet on every single job + # run. It's like having a local library of all the third-party code lnd + # needs. - name: go module cache if: ${{ inputs.use-build-cache == 'no' }} uses: actions/cache@v4 @@ -50,9 +80,8 @@ runs: # Just the module download cache. path: | ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache-${{ github.job }}- ${{ runner.os }}-go-${{ inputs.go-version }}-${{ inputs.key-prefix }}-no-build-cache- - name: set GOPATH diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6046f3efd..34675e827 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,6 +11,12 @@ on: branches: - "master" +permissions: + # Required to manage and delete caches. + actions: write + # Default permission for checking out code. + contents: read + concurrency: # Cancel any previous workflows if they are from a PR or push. group: ${{ github.event.pull_request.number || github.ref }} @@ -37,60 +43,67 @@ env: GO_VERSION: 1.23.10 jobs: - ######################## - # SQLC code gen check - ######################## - sqlc-check: - name: Sqlc check + static-checks: + name: Static Checks runs-on: ubuntu-latest steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 + with: + # Needed for some checks. + fetch-depth: 0 - - name: setup go ${{ env.GO_VERSION }} + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Setup Go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' + use-build-cache: 'no' - - name: docker image cache + ######################## + # sample configuration check + ######################## + - name: Check default values in sample-lnd.conf file + run: make sample-conf-check + + ######################## + # Check code and RPC format + ######################## + - name: Check code format + run: make fmt-check + + - name: Check go modules tidiness + run: make tidy-module-check + + - name: Lint proto files + run: make protolint + + ######################## + # SQLC code gen check + ######################## + - name: Docker image cache uses: satackey/action-docker-layer-caching@v0.0.11 # Ignore the failure of a step and avoid terminating the job. continue-on-error: true - - name: Generate sql models + - name: Check SQL models run: make sqlc-check - ######################## - # RPC and mobile compilation check - ######################## - rpc-check: - name: RPC and mobile compilation check - runs-on: ubuntu-latest - steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout - uses: actions/checkout@v4 - - - name: setup go ${{ env.GO_VERSION }} - uses: ./.github/actions/setup-go - with: - go-version: '${{ env.GO_VERSION }}' - - - name: run check + ######################## + # RPC and mobile compilation check + ######################## + - name: Check RPC format run: make rpc-check - - name: run JSON/WASM stub compilation check + - name: Check JSON/WASM stub compilation run: make rpc-js-compile - - name: build mobile RPC bindings + - name: Check mobile RPC bindings run: make mobile-rpc - - name: build mobile specific code + - name: Check mobile specific code run: go build --tags="mobile" ./mobile ######################## @@ -98,21 +111,23 @@ jobs: ######################## check-commits: if: github.event_name == 'pull_request' - name: check commits + name: Check commits runs-on: ubuntu-latest steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' + # Use the same cache from unit test job to save time. + key-prefix: unit-test - name: fetch and rebase on ${{ github.base_ref }} uses: ./.github/actions/rebase @@ -124,30 +139,23 @@ jobs: # lint code ######################## lint: - name: lint code + name: Lint code runs-on: ubuntu-latest steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - name: git checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + - name: setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' - - - name: check code format - run: make fmt-check - - - name: check go modules tidiness - run: make tidy-module-check - - - name: lint proto files - run: make protolint + # Use the same cache from unit test job to save time. + key-prefix: unit-test - name: lint run: GOGC=50 make lint @@ -156,7 +164,7 @@ jobs: # cross compilation ######################## cross-compile: - name: cross compilation + name: Cross compilation runs-on: ubuntu-latest strategy: fail-fast: true @@ -170,48 +178,27 @@ jobs: - name: arm sys: darwin-arm64 freebsd-arm linux-armv6 linux-armv7 linux-arm64 windows-arm steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - - name: setup go ${{ env.GO_VERSION }} + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: cross-compile use-build-cache: 'no' - - name: build release for all architectures + - name: Build release for all architectures run: make release sys="${{ matrix.sys }}" - ######################## - # sample configuration check - ######################## - sample-conf-check: - name: sample configuration check - runs-on: ubuntu-latest - steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout - uses: actions/checkout@v4 - - - name: setup go ${{ env.GO_VERSION }} - uses: ./.github/actions/setup-go - with: - go-version: '${{ env.GO_VERSION }}' - - - name: check default values in sample-lnd.conf file - run: make sample-conf-check - ######################## # run unit tests ######################## unit-test: - name: run unit tests + name: Run unit tests runs-on: ubuntu-latest strategy: # Allow other tests in the matrix to continue if one fails. @@ -228,37 +215,37 @@ jobs: - unit-module steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: git checkout fuzzing seeds + - name: Git checkout fuzzing seeds uses: actions/checkout@v4 with: repository: lightninglabs/lnd-fuzz path: lnd-fuzz - - name: rsync fuzzing seeds + - name: Rsync fuzzing seeds run: rsync -a --ignore-existing lnd-fuzz/ ./ - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: unit-test - - name: install bitcoind + - name: Install bitcoind run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION - - name: run ${{ matrix.unit_type }} + - name: Run ${{ matrix.unit_type }} run: make ${{ matrix.unit_type }} - name: Clean coverage @@ -280,7 +267,7 @@ jobs: # run integration tests with TRANCHES ######################## basic-integration-test: - name: basic itests + name: Run basic itests runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' strategy: @@ -297,28 +284,28 @@ jobs: - name: neutrino args: backend=neutrino cover=1 steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: install bitcoind + - name: Install bitcoind run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION - - name: run ${{ matrix.name }} + - name: Run ${{ matrix.name }} run: make itest-parallel tranches=${{ env.TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }} - name: Clean coverage @@ -352,7 +339,7 @@ jobs: # run integration tests with SMALL_TRANCHES ######################## integration-test: - name: itests + name: Run itests runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' strategy: @@ -377,25 +364,28 @@ jobs: - name: bitcoind-postgres-nativesql-experiment args: backend=bitcoind dbbackend=postgres nativesql=true tags=test_native_sql steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: install bitcoind + - name: Install bitcoind run: ./scripts/install_bitcoind.sh $BITCOIN_VERSION - - name: run ${{ matrix.name }} + - name: Run ${{ matrix.name }} run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} ${{ matrix.args }} shuffleseed=${{ github.run_id }}${{ strategy.job-index }} - name: Clean coverage @@ -430,29 +420,29 @@ jobs: # run windows integration test ######################## windows-integration-test: - name: windows itest + name: Run windows itest runs-on: windows-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: run itest + - name: Run itest run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} windows=1 shuffleseed=${{ github.run_id }} - - name: kill any remaining lnd processes + - name: Kill any remaining lnd processes if: ${{ failure() }} shell: powershell run: taskkill /IM lnd-itest.exe /T /F @@ -474,26 +464,26 @@ jobs: # run macOS integration test ######################## macos-integration-test: - name: macOS itest + name: Run macOS itest runs-on: macos-14 if: '!contains(github.event.pull_request.labels.*.name, ''no-itest'')' steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: fetch and rebase on ${{ github.base_ref }} + - name: Fetch and rebase on ${{ github.base_ref }} if: github.event_name == 'pull_request' uses: ./.github/actions/rebase - - name: setup go ${{ env.GO_VERSION }} + - name: Setup go ${{ env.GO_VERSION }} uses: ./.github/actions/setup-go with: go-version: '${{ env.GO_VERSION }}' key-prefix: integration-test - - name: run itest + - name: Run itest run: make itest-parallel tranches=${{ env.SMALL_TRANCHES }} shuffleseed=${{ github.run_id }} - name: Zip log files on failure @@ -513,7 +503,7 @@ jobs: # check pinned dependencies ######################## dep-pin: - name: check pinned dependencies + name: Check pinned dependencies runs-on: ubuntu-latest strategy: # Allow other tests in the matrix to continue if one fails. @@ -524,53 +514,95 @@ jobs: - github.com/golang/protobuf v1.5.3 steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - - name: ensure dependencies at correct version + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Ensure dependencies at correct version run: if ! grep -q "${{ matrix.pinned_dep }}" go.mod; then echo dependency ${{ matrix.pinned_dep }} should not be altered ; exit 1 ; fi ######################## # check PR updates release notes ######################## milestone-check: - name: check release notes updated + name: Check release notes updated runs-on: ubuntu-latest if: '!contains(github.event.pull_request.labels.*.name, ''no-changelog'')' steps: - - name: cleanup space - run: rm -rf /opt/hostedtoolcache - - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - - name: release notes check + - name: Clean up runner space + uses: ./.github/actions/cleanup-space + + - name: Release notes check run: scripts/check-release-notes.sh ######################## # Backwards Compatibility Test ######################## - backwards-compatability-test: - name: backwards compatability test + backwards-compatibility-test: + name: Backwards compatibility test runs-on: ubuntu-latest steps: - - name: git checkout + - name: Git checkout uses: actions/checkout@v4 - name: 🐳 Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: 🛡️ backwards compatibility test + - name: 🛡️ Backwards compatibility test run: make backwards-compat-test + ######################################### + # Auto Cache Cleanup on Pull Requests + ######################################### + auto-cleanup-cache: + name: Cache Cleanup + runs-on: ubuntu-latest + + # This condition checks for pull requests from authors with write access. + if: >- + contains('OWNER, MEMBER, COLLABORATOR', github.event.pull_request.author_association) + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Delete caches older than 12 hours + continue-on-error: true + env: + # GITHUB_TOKEN is required for the gh CLI. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Finding caches not used in the last 12 hours..." + + # Get the current time and the cutoff time (12 hours ago) in Unix + # timestamp format. + cutoff_timestamp=$(date -d "12 hours ago" +%s) + + # Use gh and jq to parse caches. Delete any cache last accessed + # before the cutoff time. + gh cache list --json key,lastAccessedAt | jq -r '.[] | + select(.lastAccessedAt != null) | "\(.lastAccessedAt) \(.key)"' | + while read -r last_accessed_at key; do + last_accessed_timestamp=$(date -d "$last_accessed_at" +%s) + + if (( last_accessed_timestamp < cutoff_timestamp )); then + echo "Deleting old cache. Key: $key, Last Used: $last_accessed_at" + gh cache delete "$key" + fi + done + # Notify about the completion of all coverage collecting jobs. finish: - if: ${{ always() }} + name: Send coverage report + if: ${{ !cancelled() }} needs: [unit-test, basic-integration-test] runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Send coverage uses: coverallsapp/github-action@v2 diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index b3969775a..62b79f9b0 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -4,12 +4,21 @@ on: pull_request: types: [opened] +permissions: + # Required to post stats as comments. + actions: write + # Default permission for checking out code. + contents: read + jobs: stats: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write + + # Check if the PR is from the base repo (not a fork). Only the + # collaborators have the permission to create a side branch from the base + # repo, so this implicitly restricts who can run this job. + if: github.event.pull_request.head.repo.fork == false + steps: - name: Run pull request stats uses: flowwer-dev/pull-request-stats@v2.11.0