diff --git a/.editorconfig b/.editorconfig index c5f3028c503..d7fe7ad5e24 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,rs,py,sh}] indent_size = 4 -# .cirrus.yml, etc. +# ci.yml, etc. [*.yml] indent_size = 2 diff --git a/.github/actions/restore-caches/action.yml b/.github/actions/cache/restore/action.yml similarity index 77% rename from .github/actions/restore-caches/action.yml rename to .github/actions/cache/restore/action.yml index 21f2807f4c7..2cc5b53a1e1 100644 --- a/.github/actions/restore-caches/action.yml +++ b/.github/actions/cache/restore/action.yml @@ -1,43 +1,51 @@ name: 'Restore Caches' description: 'Restore ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: - name: Restore Ccache cache id: ccache-cache - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} restore-keys: | ccache-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore depends sources cache id: depends-sources - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-sources-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore built depends cache id: depends-built - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-built-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore previous releases cache id: previous-releases - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} restore-keys: | previous-releases-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: export cache hits shell: bash diff --git a/.github/actions/cache/restore/internal/action.yml b/.github/actions/cache/restore/internal/action.yml new file mode 100644 index 00000000000..43dd206345d --- /dev/null +++ b/.github/actions/cache/restore/internal/action.yml @@ -0,0 +1,43 @@ +name: 'Cache Restore' +description: 'Restore a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to restore' + required: true + key: + description: 'An explicit key for restoring the cache' + required: true + restore-keys: + description: 'An ordered multiline string listing prefix-matched restore keys' + required: false + default: '' + provider: + description: 'The cache provider to use' + required: true +outputs: + cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' + value: ${{ steps.warp.outputs.cache-hit || steps.gha.outputs.cache-hit }} + cache-primary-key: + description: 'The primary key used to restore the cache' + value: ${{ steps.warp.outputs.cache-primary-key || steps.gha.outputs.cache-primary-key }} +runs: + using: 'composite' + steps: + - name: Restore cache with WarpBuild + id: warp + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/restore@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} + + - name: Restore cache with GitHub Actions + id: gha + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} diff --git a/.github/actions/save-caches/action.yml b/.github/actions/cache/save/action.yml similarity index 79% rename from .github/actions/save-caches/action.yml rename to .github/actions/cache/save/action.yml index 3072ab3f224..5c543b3f9e7 100644 --- a/.github/actions/save-caches/action.yml +++ b/.github/actions/cache/save/action.yml @@ -1,5 +1,9 @@ name: 'Save Caches' description: 'Save ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: @@ -11,29 +15,33 @@ runs: echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}" - name: Save Ccache cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }} with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} + provider: ${{ inputs.provider }} - name: Save depends sources cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }} with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save built depends cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}} with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save previous releases cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}} with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} + provider: ${{ inputs.provider }} diff --git a/.github/actions/cache/save/internal/action.yml b/.github/actions/cache/save/internal/action.yml new file mode 100644 index 00000000000..12f4156a072 --- /dev/null +++ b/.github/actions/cache/save/internal/action.yml @@ -0,0 +1,28 @@ +name: 'Cache Save' +description: 'Save a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to cache' + required: true + key: + description: 'An explicit key for saving the cache' + required: true + provider: + description: 'The cache provider to use' + required: true +runs: + using: 'composite' + steps: + - name: Save cache with WarpBuild + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/save@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + + - name: Save cache with GitHub Actions + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/save@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} diff --git a/.github/actions/configure-docker/action.yml b/.github/actions/configure-docker/action.yml index b4abf7c274a..074b47ce348 100644 --- a/.github/actions/configure-docker/action.yml +++ b/.github/actions/configure-docker/action.yml @@ -1,27 +1,27 @@ name: 'Configure Docker' description: 'Set up Docker build driver and configure build cache args' inputs: - cache-provider: - description: 'gha or cirrus cache provider' - required: true + provider: + description: 'The cache provider to use' + required: false + default: 'gha' runs: using: 'composite' steps: - - name: Check inputs - shell: python - run: | - # We expect only gha or cirrus as inputs to cache-provider - if "${{ inputs.cache-provider }}" not in ("gha", "cirrus"): - print("::warning title=Unknown input to configure docker action::Provided value was ${{ inputs.cache-provider }}") - - - name: Set up Docker Buildx + - name: Set up Docker Buildx for Warp cache + if: ${{ inputs.provider == 'warp' }} uses: docker/setup-buildx-action@v4 with: - # Use host network to allow access to cirrus gha cache running on the host driver-opts: | network=host - # This is required to allow buildkit to access the actions cache + - name: Set up Docker Buildx + if: ${{ inputs.provider != 'warp' }} + uses: docker/setup-buildx-action@v4 + + # This is required when using the gha cache backend with a manual docker buildx invocation. + # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN + # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - name: Expose actions cache variables uses: actions/github-script@v8 with: @@ -36,25 +36,18 @@ runs: - name: Construct docker build cache args shell: bash run: | - # Configure docker build cache backend - # - # On forks the gha cache will work but will use Github's cache backend. - # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN - # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - - # Use cirrus cache host - if [[ ${{ inputs.cache-provider }} == 'cirrus' ]]; then - url_args="url=${CIRRUS_CACHE_HOST},url_v2=${CIRRUS_CACHE_HOST}" - else - url_args="" + cache_options="scope=${CONTAINER_NAME}" + if [[ "${{ inputs.provider }}" == "warp" ]]; then + cache_options="url=http://127.0.0.1:49160/,version=1,${cache_options}" fi + # Configure docker build cache backend # Always optimistically --cache‑from in case a cache blob exists - args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") + args=(--cache-from "type=gha,${cache_options}") - # Only add --cache-to when using the Cirrus cache provider and pushing to the default branch. - if [[ ${{ inputs.cache-provider }} == 'cirrus' && ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then - args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}") + # Only add --cache-to when pushing to the default branch. + if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then + args+=(--cache-to "type=gha,mode=max,ignore-error=true,${cache_options}") fi # Always `--load` into docker images (needed when using the `docker-container` build driver). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 914b29cad3d..4d3a1b02bf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,7 @@ concurrency: env: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type. - REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners + REPO_USE_WARP_RUNNERS: 'bitcoin/bitcoin' # Use warp runners for this repo, instead of falling back to the slow GHA runners defaults: run: @@ -47,9 +46,9 @@ jobs: fi - id: runners run: | - if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then - echo "provider=cirrus" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using Cirrus Runners" + if [[ "${REPO_USE_WARP_RUNNERS}" == "${{ github.repository }}" ]]; then + echo "provider=warp" >> "$GITHUB_OUTPUT" + echo "::notice title=Runner Selection::Using Warp Runners" else echo "provider=gha" >> "$GITHUB_OUTPUT" echo "::notice title=Runner Selection::Using GitHub-hosted runners" @@ -58,9 +57,9 @@ jobs: test-each-commit: name: 'test ancestor commits' needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }} env: - TEST_RUNNER_PORT_MIN: "14000" # Use a larger port, to avoid colliding with CIRRUS_CACHE_HOST port 12321. + TEST_RUNNER_PORT_MIN: "14000" # Use a larger port range to avoid colliding with other CI services. if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. steps: @@ -332,7 +331,7 @@ jobs: windows-cross: name: 'Windows-cross to x86_64, ${{ matrix.crt }}' needs: [runners, record-frozen-commit] - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-4x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} strategy: @@ -364,18 +363,22 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ needs.runners.outputs.provider }} - name: Configure Docker uses: ./.github/actions/configure-docker with: - cache-provider: ${{ needs.runners.outputs.provider }} + provider: ${{ needs.runners.outputs.provider }} - name: CI script run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ needs.runners.outputs.provider }} - name: Upload built executables uses: actions/upload-artifact@v7 @@ -443,7 +446,7 @@ jobs: ci-matrix: name: ${{ matrix.name }} needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && matrix.cirrus-runner || matrix.fallback-runner }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && matrix.warp-runner || matrix.fallback-runner }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: ${{ matrix.timeout-minutes }} @@ -456,87 +459,87 @@ jobs: matrix: include: - name: 'iwyu' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_iwyu.sh' - name: '32 bit ARM' - cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode + warp-runner: 'ubuntu-24.04-arm' # Warp's Arm runners don't support 32-bit mode currently fallback-runner: 'ubuntu-24.04-arm' timeout-minutes: 120 file-env: './ci/test/00_setup_env_arm.sh' provider: 'gha' - name: 'ASan + LSan + UBSan + integer' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools + warp-runner: 'warp-ubuntu-2404-x64-8x' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_asan.sh' - name: 'macOS-cross to arm64' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross.sh' - name: 'macOS-cross to x86_64' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross_intel.sh' - name: 'No wallet' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_nowallet.sh' - name: 'i686, no IPC' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_i686_no_ipc.sh' - name: 'fuzzer,address,undefined,integer' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 240 file-env: './ci/test/00_setup_env_native_fuzz.sh' - name: 'previous releases' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_previous_releases.sh' - name: 'Alpine (musl)' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_alpine_musl.sh' - name: 'tidy' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tidy.sh' - name: 'TSan' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tsan.sh' - name: 'MSan, fuzz' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 150 file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh' - name: 'MSan' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_msan.sh' @@ -550,12 +553,14 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Configure Docker uses: ./.github/actions/configure-docker with: - cache-provider: ${{ matrix.provider || needs.runners.outputs.provider }} + provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Clear unnecessary files if: ${{ needs.runners.outputs.provider == 'gha' && true || false }} # Only needed on GHA runners @@ -576,12 +581,14 @@ jobs: run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ matrix.provider || needs.runners.outputs.provider }} lint: name: 'lint' needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-2x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: @@ -598,7 +605,7 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker with: - cache-provider: ${{ needs.runners.outputs.provider }} + provider: ${{ needs.runners.outputs.provider }} - name: CI script run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 892f30d94bf..ed30fdbb7d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,9 +28,9 @@ get_directory_property(precious_variables CACHE_VARIABLES) #============================= set(CLIENT_NAME "Bitcoin Core") set(CLIENT_VERSION_MAJOR 31) -set(CLIENT_VERSION_MINOR 0) +set(CLIENT_VERSION_MINOR 1) set(CLIENT_VERSION_BUILD 0) -set(CLIENT_VERSION_RC 0) +set(CLIENT_VERSION_RC 1) set(CLIENT_VERSION_IS_RELEASE "true") set(COPYRIGHT_YEAR "2026") diff --git a/ci/README.md b/ci/README.md index 293294a559c..0b6a34460ca 100644 --- a/ci/README.md +++ b/ci/README.md @@ -80,20 +80,22 @@ trigger cache-invalidation and rebuilds as necessary. To configure the primary repository, follow these steps: -1. Register with [Cirrus Runners](https://cirrus-runners.app/) and purchase runners. -2. Install the Cirrus Runners GitHub app against the GitHub organization. +1. Register with [WarpBuild](https://www.warpbuild.com/) and purchase runners. +2. Install the WarpBuild GitHub app against the GitHub organization. 3. Enable organisation-level runners to be used in public repositories: 1. `Org settings -> Actions -> Runner Groups -> Default -> Allow public repos` 4. Permit the following actions to run: - 1. cirruslabs/cache/restore@\* - 1. cirruslabs/cache/save@\* - 1. docker/setup-buildx-action@\* + 1. actions/cache/restore@\* + 1. actions/cache/save@\* 1. actions/github-script@\* + 1. docker/setup-buildx-action@\* + 1. warpbuilds/cache/restore@\* + 1. warpbuilds/cache/save@\* ### Forked repositories When used in a fork the CI will run on GitHub's free hosted runners by default. -In this case, due to GitHub's 10GB-per-repo cache size limitations caches will be frequently evicted and missed, but the workflows will run (slowly). +In this case, GitHub's cache size limitations may cause caches to be frequently evicted and missed, but the workflows will run (slowly). -It is also possible to use your own Cirrus Runners in your own fork with an appropriate patch to the `REPO_USE_CIRRUS_RUNNERS` variable in ../.github/workflows/ci.yml -NB that Cirrus Runners only work at an organisation level, therefore in order to use your own Cirrus Runners, *the fork must be within your own organisation*. +It is also possible to use your own WarpBuild Runners in your own fork with an appropriate patch to the `REPO_USE_WARP_RUNNERS` variable in ../.github/workflows/ci.yml +NB that WarpBuild Runners only work at an organisation level, therefore in order to use your own WarpBuild Runners, *the fork must be within your own organisation*. diff --git a/doc/man/bitcoin-cli.1 b/doc/man/bitcoin-cli.1 index 721a470c713..f8fc3d543f3 100644 --- a/doc/man/bitcoin-cli.1 +++ b/doc/man/bitcoin-cli.1 @@ -1,7 +1,7 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIN-CLI "1" "April 2026" "bitcoin-cli v31.0.0" "User Commands" +.TH BITCOIN-CLI "1" "June 2026" "bitcoin-cli v31.1.0rc1" "User Commands" .SH NAME -bitcoin-cli \- manual page for bitcoin-cli v31.0.0 +bitcoin-cli \- manual page for bitcoin-cli v31.1.0rc1 .SH SYNOPSIS .B bitcoin-cli [\fI\,options\/\fR] \fI\, \/\fR[\fI\,params\/\fR] @@ -15,7 +15,7 @@ bitcoin-cli \- manual page for bitcoin-cli v31.0.0 .B bitcoin-cli [\fI\,options\/\fR] \fI\,help \/\fR .SH DESCRIPTION -Bitcoin Core RPC client version v31.0.0 +Bitcoin Core RPC client version v31.1.0rc1 .PP The bitcoin\-cli utility provides a command line interface to interact with a Bitcoin Core RPC server. .PP diff --git a/doc/man/bitcoin-qt.1 b/doc/man/bitcoin-qt.1 index aa90489c1fd..87d55bd8769 100644 --- a/doc/man/bitcoin-qt.1 +++ b/doc/man/bitcoin-qt.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIN-QT "1" "April 2026" "bitcoin-qt v31.0.0" "User Commands" +.TH BITCOIN-QT "1" "June 2026" "bitcoin-qt v31.1.0rc1" "User Commands" .SH NAME -bitcoin-qt \- manual page for bitcoin-qt v31.0.0 +bitcoin-qt \- manual page for bitcoin-qt v31.1.0rc1 .SH SYNOPSIS .B bitcoin-qt [\fI\,options\/\fR] [\fI\,URI\/\fR] .SH DESCRIPTION -Bitcoin Core version v31.0.0 +Bitcoin Core version v31.1.0rc1 .PP The bitcoin\-qt application provides a graphical interface for interacting with Bitcoin Core. .PP diff --git a/doc/man/bitcoin-tx.1 b/doc/man/bitcoin-tx.1 index 8ecea692f77..7d14074eeda 100644 --- a/doc/man/bitcoin-tx.1 +++ b/doc/man/bitcoin-tx.1 @@ -1,7 +1,7 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIN-TX "1" "April 2026" "bitcoin-tx v31.0.0" "User Commands" +.TH BITCOIN-TX "1" "June 2026" "bitcoin-tx v31.1.0rc1" "User Commands" .SH NAME -bitcoin-tx \- manual page for bitcoin-tx v31.0.0 +bitcoin-tx \- manual page for bitcoin-tx v31.1.0rc1 .SH SYNOPSIS .B bitcoin-tx [\fI\,options\/\fR] \fI\, \/\fR[\fI\,commands\/\fR] @@ -9,7 +9,7 @@ bitcoin-tx \- manual page for bitcoin-tx v31.0.0 .B bitcoin-tx [\fI\,options\/\fR] \fI\,-create \/\fR[\fI\,commands\/\fR] .SH DESCRIPTION -Bitcoin Core bitcoin\-tx utility version v31.0.0 +Bitcoin Core bitcoin\-tx utility version v31.1.0rc1 .PP The bitcoin\-tx tool is used for creating and modifying bitcoin transactions. .PP diff --git a/doc/man/bitcoin-util.1 b/doc/man/bitcoin-util.1 index 6e7b1be2c06..7dfe41473d2 100644 --- a/doc/man/bitcoin-util.1 +++ b/doc/man/bitcoin-util.1 @@ -1,7 +1,7 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIN-UTIL "1" "April 2026" "bitcoin-util v31.0.0" "User Commands" +.TH BITCOIN-UTIL "1" "June 2026" "bitcoin-util v31.1.0rc1" "User Commands" .SH NAME -bitcoin-util \- manual page for bitcoin-util v31.0.0 +bitcoin-util \- manual page for bitcoin-util v31.1.0rc1 .SH SYNOPSIS .B bitcoin-util [\fI\,options\/\fR] [\fI\,command\/\fR] @@ -9,7 +9,7 @@ bitcoin-util \- manual page for bitcoin-util v31.0.0 .B bitcoin-util [\fI\,options\/\fR] \fI\,grind \/\fR .SH DESCRIPTION -Bitcoin Core bitcoin\-util utility version v31.0.0 +Bitcoin Core bitcoin\-util utility version v31.1.0rc1 .PP The bitcoin\-util tool provides bitcoin related functionality that does not rely on the ability to access a running node. Available [commands] are listed below. .SH OPTIONS diff --git a/doc/man/bitcoin-wallet.1 b/doc/man/bitcoin-wallet.1 index 33ea607c672..4486d99de15 100644 --- a/doc/man/bitcoin-wallet.1 +++ b/doc/man/bitcoin-wallet.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIN-WALLET "1" "April 2026" "bitcoin-wallet v31.0.0" "User Commands" +.TH BITCOIN-WALLET "1" "June 2026" "bitcoin-wallet v31.1.0rc1" "User Commands" .SH NAME -bitcoin-wallet \- manual page for bitcoin-wallet v31.0.0 +bitcoin-wallet \- manual page for bitcoin-wallet v31.1.0rc1 .SH SYNOPSIS .B bitcoin-wallet [\fI\,options\/\fR] \fI\,\/\fR .SH DESCRIPTION -Bitcoin Core bitcoin\-wallet utility version v31.0.0 +Bitcoin Core bitcoin\-wallet utility version v31.1.0rc1 .PP bitcoin\-wallet is an offline tool for creating and interacting with Bitcoin Core wallet files. .PP diff --git a/doc/man/bitcoin.1 b/doc/man/bitcoin.1 index 78c93deeea3..d6d1aee9795 100644 --- a/doc/man/bitcoin.1 +++ b/doc/man/bitcoin.1 @@ -1,7 +1,7 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIN "1" "April 2026" "bitcoin v31.0.0" "User Commands" +.TH BITCOIN "1" "June 2026" "bitcoin v31.1.0rc1" "User Commands" .SH NAME -bitcoin \- manual page for bitcoin v31.0.0 +bitcoin \- manual page for bitcoin v31.1.0rc1 .SH SYNOPSIS .B bitcoin [\fI\,OPTIONS\/\fR] \fI\,COMMAND\/\fR... diff --git a/doc/man/bitcoind.1 b/doc/man/bitcoind.1 index e42dcd7f7c0..68975b126e5 100644 --- a/doc/man/bitcoind.1 +++ b/doc/man/bitcoind.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH BITCOIND "1" "April 2026" "bitcoind v31.0.0" "User Commands" +.TH BITCOIND "1" "June 2026" "bitcoind v31.1.0rc1" "User Commands" .SH NAME -bitcoind \- manual page for bitcoind v31.0.0 +bitcoind \- manual page for bitcoind v31.1.0rc1 .SH SYNOPSIS .B bitcoind [\fI\,options\/\fR] .SH DESCRIPTION -Bitcoin Core daemon version v31.0.0 bitcoind +Bitcoin Core daemon version v31.1.0rc1 bitcoind .PP The Bitcoin Core daemon (bitcoind) is a headless program that connects to the Bitcoin network to validate and relay transactions and blocks, as well as relaying addresses. .PP diff --git a/doc/release-notes.md b/doc/release-notes.md index a1b45a0bd66..d70bb5167b8 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,9 +1,9 @@ -v31.x Release Notes +v31.1rc1 Release Notes =================== -Bitcoin Core version 31.x is now available from: +Bitcoin Core version 31.1rc1 is now available from: - + This release includes new features, various bug fixes and performance improvements, as well as updated translations. @@ -39,17 +39,35 @@ on them. It is not recommended to use Bitcoin Core on unsupported systems. Notable changes =============== +### PrivateBroadcast + +This release fixes an ip address leak when using the -privatebroadcast feature. +Under certain circumstances connections were being made over clearnet rather than +the enabled privacy network. + + ### Validation - #35209 validation: correct lifetime of precomputed tx data +- #35465 coins: compact chainstate regularly + +### Leveldb + +- #61(bitcoin-core/leveldb): Disable seek compaction ### P2P - #35032 net_processing: don't modify addrman for private broadcast connections +- #35410 net: use the proxy if overriden when doing v2->v1 reconnections ### Wallet - #35227 wallet: check the final BDB page LSN during migration +- #35228 wallet: use outpoint when estimating input size + +### Musig + +- #35316 musig: Reject empty pubkey list in GetMuSig2KeyAggCache ### Build @@ -62,11 +80,16 @@ Notable changes - #34991 test: fix feature_index_prune.py bug when using --usecli - #35080 test: Add missing self.options.timeout_factor scale in tool_bitcoin_chainstate.py - #35218 test: fix P2SH script in coins cache fuzz target +- #35279 psbt, test: remove address type restrictions in test ### Fuzz - #35289 fuzz: Fix timeout in txorphan +### Util + +- #35384 util: Check write failures before renaming settings.json + ### Docs - #35283 doc: mention -DWITH_ZMQ=ON in BSD build guides @@ -75,29 +98,43 @@ Notable changes - #35202 ci: restore sockets in i686, no IPC job - #35230 ci: Move --usecli --extended from i386 task to alpine task +- #35348 ci: switch to GitHub cache for all runners +- #35378 ci: switch runners from cirrus to warpbuild +- #35408 ci: 35378 followups +- #35430 ci: use warp caching on warp runners +- #35447 ci: use warpbuild cache for docker buildkit cache ### Misc - #35044 contrib: Fix NameError in signet miner gbt() - #35175 multi_index: fix compilation failure with boost >= 1.91 +- #34953 crypto: disable ASan instrumentation of SSE4 SHA256 for GCC Credits ======= Thanks to everyone who directly contributed to this release: +- andrewtoth - Cory Fields +- Crypt-iQ - darosior +- deadmanoz +- fanquake - Greg Sanders - Hennadii Stepanov - junbyjun1238 - Lőrinc - MarcoFalke - marcofleon +- nervana21 - optout21 - Pol Espinasa +- rkrux +- Shrey - Torkel Rogstad - Vasil Dimov +- willcl-ark As well as to everyone that helped with translations on [Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/src/common/settings.cpp b/src/common/settings.cpp index e9b929794ff..7d511b574de 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -86,7 +86,7 @@ bool ReadSettings(const fs::path& path, std::map& va SettingsValue in; if (!in.read(std::string{std::istreambuf_iterator(file), std::istreambuf_iterator()})) { - errors.emplace_back(strprintf("Settings file %s does not contain valid JSON. This is probably caused by disk corruption or a crash, " + errors.emplace_back(strprintf("Settings file %s does not contain valid JSON. This may be caused by a crash, power loss, full disk, or storage error, " "and can be fixed by removing the file, which will reset settings to default values.", fs::PathToString(path))); return false; @@ -139,7 +139,15 @@ bool WriteSettings(const fs::path& path, return false; } file << out.write(/* prettyIndent= */ 4, /* indentLevel= */ 1) << std::endl; + if (file.fail()) { + errors.emplace_back(strprintf("Error: Unable to write settings file %s", fs::PathToString(path))); + return false; + } file.close(); + if (file.fail()) { + errors.emplace_back(strprintf("Error: Unable to close settings file %s", fs::PathToString(path))); + return false; + } return true; } diff --git a/src/crypto/sha256_sse4.cpp b/src/crypto/sha256_sse4.cpp index 2d37d124d1e..4464ec92439 100644 --- a/src/crypto/sha256_sse4.cpp +++ b/src/crypto/sha256_sse4.cpp @@ -12,18 +12,23 @@ namespace sha256_sse4 { -void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) -#if defined(__clang__) - /* - clang is unable to compile this with -O0 and -fsanitize=address. - See upstream bug: https://github.com/llvm/llvm-project/issues/92182. - This also fails to compile with -O2, -fcf-protection & -fsanitize=address. - See https://github.com/bitcoin/bitcoin/issues/31913. - */ -#if __has_feature(address_sanitizer) +/* +Both Clang and GCC fail with ASan on this inline assembly: +- Clang: compile failure with -O0 or -O2 + -fcf-protection under ASan. + See https://github.com/llvm/llvm-project/issues/92182 + and https://github.com/bitcoin/bitcoin/issues/31913. +- GCC: runtime SEGV during SHA256AutoDetect()'s self-test under ASan, + regardless of optimization level. + See https://github.com/bitcoin/bitcoin/issues/34881. +*/ +#if defined(__SANITIZE_ADDRESS__) + __attribute__((no_sanitize("address"))) +#elif defined(__clang__) +#if __has_feature(address_sanitizer) // fallback can be removed once support for Clang 21 is dropped __attribute__((no_sanitize("address"))) #endif #endif +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) { static const uint32_t K256 alignas(16) [] = { 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index eb222078b5e..6a3f6b00f13 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -246,7 +246,7 @@ CDBWrapper::CDBWrapper(const DBParams& params) if (params.options.force_compact) { LogInfo("Starting database compaction of %s", fs::PathToString(params.path)); - DBContext().pdb->CompactRange(nullptr, nullptr); + CompactFull(); LogInfo("Finished database compaction of %s", fs::PathToString(params.path)); } @@ -291,11 +291,18 @@ void CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) } } +std::optional CDBWrapper::GetProperty(const std::string& property) const +{ + if (std::string value; DBContext().pdb->GetProperty(property, &value)) return value; + return std::nullopt; +} + +void CDBWrapper::CompactFull() { DBContext().pdb->CompactRange(nullptr, nullptr); } + size_t CDBWrapper::DynamicMemoryUsage() const { - std::string memory; std::optional parsed; - if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral(memory))) { + if (auto memory{GetProperty("leveldb.approximate-memory-usage")}; !memory || !(parsed = ToIntegral(*memory))) { LogDebug(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n"); return 0; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 2eee6c1c023..3a3be0001cd 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -250,6 +250,12 @@ public: void WriteBatch(CDBBatch& batch, bool fSync = false); + //! Perform a blocking full compaction of the underlying LevelDB. + void CompactFull(); + + //! Return a LevelDB property value, if available. + std::optional GetProperty(const std::string& property) const; + // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; diff --git a/src/leveldb/db/autocompact_test.cc b/src/leveldb/db/autocompact_test.cc index e6c97a05a6b..f6e714c6524 100644 --- a/src/leveldb/db/autocompact_test.cc +++ b/src/leveldb/db/autocompact_test.cc @@ -54,8 +54,8 @@ static const int kValueSize = 200 * 1024; static const int kTotalSize = 100 * 1024 * 1024; static const int kCount = kTotalSize / kValueSize; -// Read through the first n keys repeatedly and check that they get -// compacted (verified by checking the size of the key space). +// Read through the first n keys repeatedly and check that reads do NOT +// trigger compaction (seek compaction is disabled in this fork). void AutoCompactTest::DoReads(int n) { std::string value(kValueSize, 'x'); DBImpl* dbi = reinterpret_cast(db_); @@ -76,25 +76,23 @@ void AutoCompactTest::DoReads(int n) { const int64_t initial_size = Size(Key(0), Key(n)); const int64_t initial_other_size = Size(Key(n), Key(kCount)); - // Read until size drops significantly. + // Read repeatedly. The size of the read range must NOT shrink: with + // seek compaction disabled, reads never schedule a compaction. std::string limit_key = Key(n); - for (int read = 0; true; read++) { - ASSERT_LT(read, 100) << "Taking too long to compact"; + for (int read = 0; read < 100; read++) { Iterator* iter = db_->NewIterator(ReadOptions()); for (iter->SeekToFirst(); iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) { // Drop data } delete iter; - // Wait a little bit to allow any triggered compactions to complete. - Env::Default()->SleepForMicroseconds(1000000); uint64_t size = Size(Key(0), Key(n)); fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1, size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0); - if (size <= initial_size / 10) { - break; - } } + // Give any background work a chance to run, even though none should. + Env::Default()->SleepForMicroseconds(1000000); + ASSERT_EQ(Size(Key(0), Key(n)), static_cast(initial_size)); // Verify that the size of the key space not touched by the reads // is pretty much unchanged. diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index 3c9f89428ff..16e8ecb9a33 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -735,15 +735,14 @@ TEST(DBTest, GetPicksCorrectFile) { } while (ChangeOptions()); } -TEST(DBTest, GetEncountersEmptyLevel) { +TEST(DBTest, GetDoesNotTriggerSeekCompaction) { do { // Arrange for the following to happen: // * sstable A in level 0 // * nothing in level 1 // * sstable B in level 2 - // Then do enough Get() calls to arrange for an automatic compaction - // of sstable A. A bug would cause the compaction to be marked as - // occurring at level 1 (instead of the correct level 0). + // Seek compaction is disabled in this fork, so repeated reads must + // not change the level layout. A manual compaction must still work. // Step 1: First place sstables in levels 0 and 2 int compaction_count = 0; @@ -761,14 +760,17 @@ TEST(DBTest, GetEncountersEmptyLevel) { ASSERT_EQ(NumTableFilesAtLevel(1), 0); ASSERT_EQ(NumTableFilesAtLevel(2), 1); - // Step 3: read a bunch of times + // Step 3: many read misses must not schedule any compaction. for (int i = 0; i < 1000; i++) { ASSERT_EQ("NOT_FOUND", Get("missing")); } - - // Step 4: Wait for compaction to finish DelayMilliseconds(1000); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 1); + // Step 4: a manual compaction still moves the L0 file down. + dbfull()->TEST_CompactRange(0, nullptr, nullptr); ASSERT_EQ(NumTableFilesAtLevel(0), 0); } while (ChangeOptions()); } diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index cd07346ea8a..8dc73295b84 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -400,16 +400,11 @@ Status Version::Get(const ReadOptions& options, const LookupKey& k, return state.found ? state.s : Status::NotFound(Slice()); } -bool Version::UpdateStats(const GetStats& stats) { - FileMetaData* f = stats.seek_file; - if (f != nullptr) { - f->allowed_seeks--; - if (f->allowed_seeks <= 0 && file_to_compact_ == nullptr) { - file_to_compact_ = f; - file_to_compact_level_ = stats.seek_file_level; - return true; - } - } +bool Version::UpdateStats(const GetStats& /*stats*/) { + // Disable automatic compactions triggered by read seek counters. + // The heuristic was tuned for expensive random seeks and can create + // severe write amplification on large random-key databases. + // Size and manual compactions still run. return false; } @@ -661,6 +656,8 @@ class VersionSet::Builder { // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. + // + // Note: seek compactions are disabled. See Version::UpdateStats. f->allowed_seeks = static_cast((f->file_size / 16384U)); if (f->allowed_seeks < 100) f->allowed_seeks = 100; diff --git a/src/musig.cpp b/src/musig.cpp index 706874be2cf..f6b579577db 100644 --- a/src/musig.cpp +++ b/src/musig.cpp @@ -16,6 +16,10 @@ constexpr uint256 MUSIG_CHAINCODE{ static bool GetMuSig2KeyAggCache(const std::vector& pubkeys, secp256k1_musig_keyagg_cache& keyagg_cache) { + if (pubkeys.empty()) { + return false; + } + // Parse the pubkeys std::vector secp_pubkeys; std::vector pubkey_ptrs; diff --git a/src/net.cpp b/src/net.cpp index 4f88aa8aab9..5c28efade89 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -487,8 +487,11 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, LogDebug(BCLog::PROXY, "Using proxy: %s to connect to %s\n", proxy.ToString(), target_addr.ToStringAddrPort()); sock = ConnectThroughProxy(proxy, target_addr.ToStringAddr(), target_addr.GetPort(), proxyConnectionFailed); } else { - // no proxy needed (none set for target network) - sock = ConnectDirectly(target_addr, conn_type == ConnectionType::MANUAL); + // No proxy needed (none set for target network). Private broadcast connections + // must always use a proxy, otherwise they would leak the originator's IP address. + if (Assume(conn_type != ConnectionType::PRIVATE_BROADCAST)) { + sock = ConnectDirectly(target_addr, conn_type == ConnectionType::MANUAL); + } } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to @@ -536,6 +539,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, network_id, CNodeOptions{ .permission_flags = permission_flags, + .proxy_override = proxy_override, .i2p_sam_session = std::move(i2p_transient_session), .recv_flood_size = nReceiveFloodSize, .use_v2transport = use_v2transport, @@ -1902,7 +1906,13 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ CountingSemaphoreGrant<> grant(*semOutbound, true); if (!grant) return false; - OpenNetworkConnection(CAddress(), false, std::move(grant), address.c_str(), conn_type, /*use_v2transport=*/use_v2transport); + OpenNetworkConnection(/*addrConnect=*/CAddress{}, + /*fCountFailure=*/false, + /*grant_outbound=*/std::move(grant), + /*pszDest=*/address.c_str(), + /*conn_type=*/conn_type, + /*use_v2transport=*/use_v2transport, + /*proxy_override=*/std::nullopt); return true; } @@ -1943,6 +1953,7 @@ void CConnman::DisconnectNodes() // and we don't want to hold up the socket handler thread for that long. if (network_active && pnode->m_transport->ShouldReconnectV1()) { reconnections_to_add.push_back({ + .proxy_override = pnode->m_proxy_override, .addr_connect = pnode->addr, .grant = std::move(pnode->grantOutbound), .destination = pnode->m_dest, @@ -2419,7 +2430,13 @@ void CConnman::ProcessAddrFetch() CAddress addr; CountingSemaphoreGrant<> grant(*semOutbound, /*fTry=*/true); if (grant) { - OpenNetworkConnection(addr, false, std::move(grant), strDest.c_str(), ConnectionType::ADDR_FETCH, use_v2transport); + OpenNetworkConnection(/*addrConnect=*/addr, + /*fCountFailure=*/false, + /*grant_outbound=*/std::move(grant), + /*pszDest=*/strDest.c_str(), + /*conn_type=*/ConnectionType::ADDR_FETCH, + /*use_v2transport=*/use_v2transport, + /*proxy_override=*/std::nullopt); } } @@ -2537,8 +2554,13 @@ void CConnman::ThreadOpenConnections(const std::vector connect, std { for (const std::string& strAddr : connect) { - CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, {}, strAddr.c_str(), ConnectionType::MANUAL, /*use_v2transport=*/use_v2transport); + OpenNetworkConnection(/*addrConnect=*/CAddress{CService{}, NODE_NONE}, + /*fCountFailure=*/false, + /*grant_outbound=*/{}, + /*pszDest=*/strAddr.c_str(), + /*conn_type=*/ConnectionType::MANUAL, + /*use_v2transport=*/use_v2transport, + /*proxy_override=*/std::nullopt); for (int i = 0; i < 10 && i < nLoop; i++) { if (!m_interrupt_net->sleep_for(500ms)) { @@ -2888,7 +2910,13 @@ void CConnman::ThreadOpenConnections(const std::vector connect, std const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(m_max_automatic_connections - 1, 2)}; // Use BIP324 transport when both us and them have NODE_V2_P2P set. const bool use_v2transport(addrConnect.nServices & GetLocalServices() & NODE_P2P_V2); - OpenNetworkConnection(addrConnect, count_failures, std::move(grant), /*pszDest=*/nullptr, conn_type, use_v2transport); + OpenNetworkConnection(/*addrConnect=*/addrConnect, + /*fCountFailure=*/count_failures, + /*grant_outbound=*/std::move(grant), + /*pszDest=*/nullptr, + /*conn_type=*/conn_type, + /*use_v2transport=*/use_v2transport, + /*proxy_override=*/std::nullopt); } } } @@ -2982,8 +3010,13 @@ void CConnman::ThreadOpenAddedConnections() break; } tried = true; - CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); + OpenNetworkConnection(/*addrConnect=*/CAddress{CService{}, NODE_NONE}, + /*fCountFailure=*/false, + /*grant_outbound=*/std::move(grant), + /*pszDest=*/info.m_params.m_added_node.c_str(), + /*conn_type=*/ConnectionType::MANUAL, + /*use_v2transport=*/info.m_params.m_use_v2transport, + /*proxy_override=*/std::nullopt); if (!m_interrupt_net->sleep_for(500ms)) return; grant = CountingSemaphoreGrant<>(*semAddnode, /*fTry=*/true); } @@ -3980,6 +4013,7 @@ CNode::CNode(NodeId idIn, m_permission_flags{node_opts.permission_flags}, m_sock{sock}, m_connected{GetTime()}, + m_proxy_override{std::move(node_opts.proxy_override)}, addr{addrIn}, addrBind{addrBindIn}, m_addr_name{addrNameIn.empty() ? addr.ToStringAddrPort() : addrNameIn}, @@ -4163,7 +4197,8 @@ void CConnman::PerformReconnections() std::move(item.grant), item.destination.empty() ? nullptr : item.destination.c_str(), item.conn_type, - item.use_v2transport); + item.use_v2transport, + item.proxy_override); } } diff --git a/src/net.h b/src/net.h index b209df61436..96a171736ed 100644 --- a/src/net.h +++ b/src/net.h @@ -669,6 +669,7 @@ public: struct CNodeOptions { NetPermissionFlags permission_flags = NetPermissionFlags::None; + std::optional proxy_override = {}; std::unique_ptr i2p_sam_session = nullptr; bool prefer_evict = false; size_t recv_flood_size{DEFAULT_MAXRECEIVEBUFFER * 1000}; @@ -711,6 +712,10 @@ public: std::atomic m_last_recv{0s}; //! Unix epoch time at peer connection const std::chrono::seconds m_connected; + + //! Proxy to use regardless of global proxy settings if reconnecting to this node. + const std::optional m_proxy_override; + // Address of this peer const CAddress addr; // Bind address of our side of the connection @@ -1186,7 +1191,7 @@ public: const char* pszDest, ConnectionType conn_type, bool use_v2transport, - const std::optional& proxy_override = std::nullopt) + const std::optional& proxy_override) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); /// Group of private broadcast related members. @@ -1778,6 +1783,7 @@ private: /** Struct for entries in m_reconnections. */ struct ReconnectionInfo { + std::optional proxy_override; CAddress addr_connect; CountingSemaphoreGrant<> grant; std::string destination; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 5c54e207973..e9fbd31cc3c 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -356,7 +356,13 @@ static RPCHelpMan addnode() if (command == "onetry") { CAddress addr; - connman.OpenNetworkConnection(addr, /*fCountFailure=*/false, /*grant_outbound=*/{}, std::string{node_arg}.c_str(), ConnectionType::MANUAL, use_v2transport); + connman.OpenNetworkConnection(/*addrConnect=*/addr, + /*fCountFailure=*/false, + /*grant_outbound=*/{}, + /*pszDest=*/std::string{node_arg}.c_str(), + /*conn_type=*/ConnectionType::MANUAL, + /*use_v2transport=*/use_v2transport, + /*proxy_override=*/std::nullopt); return UniValue::VNULL; } diff --git a/src/test/bip328_tests.cpp b/src/test/bip328_tests.cpp index 9fffa00b6b0..630771d5b6a 100644 --- a/src/test/bip328_tests.cpp +++ b/src/test/bip328_tests.cpp @@ -89,6 +89,12 @@ BOOST_AUTO_TEST_CASE(valid_keys) } } +BOOST_AUTO_TEST_CASE(empty_pubkey_list) +{ + const std::optional aggregate_pubkey{MuSig2AggregatePubkeys({})}; + BOOST_CHECK(!aggregate_pubkey.has_value()); +} + BOOST_AUTO_TEST_CASE(invalid_key) { std::vector test_vectors = { diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 8321bf6a18c..cb746a82cdd 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -1060,6 +1061,30 @@ BOOST_FIXTURE_TEST_CASE(ccoins_flush_behavior, FlushTest) } } +BOOST_FIXTURE_TEST_CASE(coins_db_leveldb_layout, FlushTest) +{ + auto level2_files{[](CCoinsViewDB& base) { + return *Assert(ToIntegral(*Assert(base.GetDBProperty("leveldb.num-files-at-level2")))); + }}; + const COutPoint outpoint{Txid::FromUint256(m_rng.rand256()), 0}; + const Coin coin{MakeCoin()}; + const uint256 block_hash{m_rng.rand256()}; + + CCoinsViewDB base{{.path = m_args.GetDataDirBase() / "coins_db_leveldb_layout", .cache_bytes = 1_MiB, .wipe_data = true}, {}}; + CCoinsViewCache cache{&base}; + + cache.EmplaceCoinInternalDANGER(COutPoint{outpoint}, Coin{coin}); + cache.SetBestBlock(block_hash); + cache.Sync(); + + BOOST_CHECK_EQUAL(level2_files(base), 0); + WITH_LOCK(::cs_main, return base.CompactFull()).wait(); + BOOST_CHECK_EQUAL(level2_files(base), 1); + + BOOST_CHECK(*Assert(base.GetCoin(outpoint)) == coin); + BOOST_CHECK_EQUAL(base.GetBestBlock(), block_hash); +} + BOOST_AUTO_TEST_CASE(coins_resource_is_used) { CCoinsMapMemoryResource resource; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 4a8b7b2155f..fdf25106525 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -173,13 +173,19 @@ FUZZ_TARGET(connman, .init = initialize_connman) conn_type = ConnectionType::OUTBOUND_FULL_RELAY; } + std::optional proxy_override; + if (conn_type == ConnectionType::PRIVATE_BROADCAST || fuzzed_data_provider.ConsumeBool()) { + proxy_override.emplace(ConsumeService(fuzzed_data_provider)); + } + connman.OpenNetworkConnection( /*addrConnect=*/random_address, /*fCountFailure=*/fuzzed_data_provider.ConsumeBool(), /*grant_outbound=*/{}, /*pszDest=*/fuzzed_data_provider.ConsumeBool() ? nullptr : random_string.c_str(), /*conn_type=*/conn_type, - /*use_v2transport=*/fuzzed_data_provider.ConsumeBool()); + /*use_v2transport=*/fuzzed_data_provider.ConsumeBool(), + /*proxy_override=*/proxy_override); }, [&] { connman.SetNetworkActive(fuzzed_data_provider.ConsumeBool()); diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 54d3b058118..0d8e3b1b7ae 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(ReadWrite) // Check invalid json not allowed WriteText(path, R"(invalid json)"); BOOST_CHECK(!common::ReadSettings(path, values, errors)); - std::vector fail_parse = {strprintf("Settings file %s does not contain valid JSON. This is probably caused by disk corruption or a crash, " + std::vector fail_parse = {strprintf("Settings file %s does not contain valid JSON. This may be caused by a crash, power loss, full disk, or storage error, " "and can be fixed by removing the file, which will reset settings to default values.", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); diff --git a/src/txdb.cpp b/src/txdb.cpp index 2c39cf2767b..9f522fb8358 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -13,10 +13,14 @@ #include #include #include +#include #include #include +#include #include +#include +#include #include #include @@ -55,11 +59,22 @@ CCoinsViewDB::CCoinsViewDB(DBParams db_params, CoinsViewOptions options) : m_options{std::move(options)}, m_db{std::make_unique(m_db_params)} { } +CCoinsViewDB::~CCoinsViewDB() +{ + if (m_compaction.valid()) { + if (m_compaction.wait_for(std::chrono::seconds{0}) != std::future_status::ready) { + LogInfo("Waiting for background chainstate compaction of %s", fs::PathToString(m_db_params.path)); + } + m_compaction.wait(); + } +} + void CCoinsViewDB::ResizeCache(size_t new_cache_size) { // We can't do this operation with an in-memory DB since we'll lose all the coins upon // reset. if (!m_db_params.memory_only) { + LOCK(m_db_mutex); // Have to do a reset first to get the original `m_db` state to release its // filesystem lock. m_db.reset(); @@ -168,6 +183,30 @@ size_t CCoinsViewDB::EstimateSize() const return m_db->EstimateSize(DB_COIN, uint8_t(DB_COIN + 1)); } +std::optional CCoinsViewDB::GetDBProperty(const std::string& property) +{ + return m_db->GetProperty(property); +} + +std::shared_future CCoinsViewDB::CompactFull() +{ + AssertLockHeld(::cs_main); + if (m_compaction.valid() && m_compaction.wait_for(std::chrono::seconds{0}) != std::future_status::ready) return m_compaction; + m_compaction = std::async(std::launch::async, [this] { + try { + util::ThreadRename("utxocompact"); + LOCK(m_db_mutex); + + LogDebug(BCLog::COINDB, "Starting chainstate compaction of %s", fs::PathToString(m_db_params.path)); + m_db->CompactFull(); + LogDebug(BCLog::COINDB, "Finished chainstate compaction of %s", fs::PathToString(m_db_params.path)); + } catch (const std::exception& e) { + LogWarning("Failed chainstate compaction (%s)", e.what()); + } + }).share(); + return m_compaction; +} + /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ class CCoinsViewDBCursor: public CCoinsViewCursor { diff --git a/src/txdb.h b/src/txdb.h index 248fe43e559..4164590c5a1 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -15,8 +15,10 @@ #include #include +#include #include #include +#include #include class COutPoint; @@ -36,9 +38,13 @@ class CCoinsViewDB final : public CCoinsView protected: DBParams m_db_params; CoinsViewOptions m_options; + //! Prevents CompactFull() from using m_db while ResizeCache() replaces it. + Mutex m_db_mutex; std::unique_ptr m_db; + std::shared_future m_compaction; public: explicit CCoinsViewDB(DBParams db_params, CoinsViewOptions options); + ~CCoinsViewDB() override; std::optional GetCoin(const COutPoint& outpoint) const override; bool HaveCoin(const COutPoint &outpoint) const override; @@ -52,7 +58,13 @@ public: size_t EstimateSize() const override; //! Dynamically alter the underlying leveldb cache size. - void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_db_mutex); + + //! Perform a full compaction of the underlying LevelDB on a one-shot background thread. + std::shared_future CompactFull() EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_db_mutex); + + //! Return an underlying LevelDB property value, if available. + std::optional GetDBProperty(const std::string& property); }; #endif // BITCOIN_TXDB_H diff --git a/src/validation.cpp b/src/validation.cpp index 23181d4b314..d3cabe44647 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -112,6 +112,13 @@ const std::vector CHECKLEVEL_DOC { * */ static constexpr int PRUNE_LOCK_BUFFER{10}; +// Return whether the completed full flush should compact chainstate +static bool ShouldCompactChainstate(bool in_ibd) +{ + static constexpr uint32_t flush_ratio{320}; // Roughly every 2 weeks with hourly flushes + return !in_ibd && FastRandomContext().randrange(flush_ratio) == 0; +} + TRACEPOINT_SEMAPHORE(validation, block_connected); TRACEPOINT_SEMAPHORE(utxocache, flush); TRACEPOINT_SEMAPHORE(mempool, replaced); @@ -2824,9 +2831,19 @@ bool Chainstate::FlushStateToDisk( m_next_write = FastRandomContext().rand_uniform_delay(NodeClock::now() + DATABASE_WRITE_INTERVAL_MIN, range); } } - if (full_flush_completed && m_chainman.m_options.signals) { - // Update best block in wallet (so we can detect restored wallets). - m_chainman.m_options.signals->ChainStateFlushed(this->GetRole(), GetLocator(m_chain.Tip())); + if (full_flush_completed) { + if (m_chainman.m_options.signals) { + // Update best block in wallet (so we can detect restored wallets). + m_chainman.m_options.signals->ChainStateFlushed(this->GetRole(), GetLocator(m_chain.Tip())); + } + + if (!m_chainman.m_interrupt && ShouldCompactChainstate(m_chainman.IsInitialBlockDownload())) { + try { + CoinsDB().CompactFull(); + } catch (const std::exception& e) { + LogWarning("Failed to start chainstate compaction (%s)", e.what()); + } + } } } catch (const std::runtime_error& e) { return FatalError(m_chainman.GetNotifications(), state, strprintf(_("System error while flushing: %s"), e.what())); diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 3d150426404..613375385d3 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -94,7 +94,7 @@ int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoin if (!provider) return -1; if (const auto desc = InferDescriptor(txout.scriptPubKey, *provider)) { - if (const auto weight = MaxInputWeight(*desc, {}, coin_control, true, can_grind_r)) { + if (const auto weight = MaxInputWeight(*desc, CTxIn{outpoint}, coin_control, true, can_grind_r)) { return static_cast(GetVirtualTransactionSize(*weight, 0, 0)); } } diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index 866ffa48c20..218792ff033 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include