diff --git a/.editorconfig b/.editorconfig index c5f3028c503..d7fe7ad5e24 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,rs,py,sh}] indent_size = 4 -# .cirrus.yml, etc. +# ci.yml, etc. [*.yml] indent_size = 2 diff --git a/.github/actions/configure-docker/action.yml b/.github/actions/configure-docker/action.yml index 9bf970ee78f..af667f0fdb5 100644 --- a/.github/actions/configure-docker/action.yml +++ b/.github/actions/configure-docker/action.yml @@ -1,7 +1,7 @@ name: 'Configure Docker' description: 'Set up Docker build driver and configure build cache args' inputs: - use-cirrus: + use-warp: description: 'Use cirrus cache' required: true runs: @@ -35,12 +35,7 @@ runs: # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - # Use cirrus cache host - if [[ ${{ inputs.use-cirrus }} == 'true' ]]; then - url_args="url=${CIRRUS_CACHE_HOST},url_v2=${CIRRUS_CACHE_HOST}" - else - url_args="" - fi + url_args="" # Always optimistically --cache‑from in case a cache blob exists args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0af3eb95fe..2a683970614 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,29 +19,29 @@ concurrency: env: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type. - REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners + REPO_USE_WARP_RUNNERS: 'bitcoin/bitcoin' # Use warp runners for this repo, instead of falling back to the slow GHA runners jobs: runners: name: 'determine runners' runs-on: ubuntu-latest outputs: - use-cirrus-runners: ${{ steps.runners.outputs.use-cirrus-runners }} + use-warp-runners: ${{ steps.runners.outputs.use-warp-runners }} steps: - id: runners run: | - if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then - echo "use-cirrus-runners=true" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using Cirrus Runners" + if [[ "${REPO_USE_WARP_RUNNERS}" == "${{ github.repository }}" ]]; then + echo "provider=warp" >> "$GITHUB_OUTPUT" + echo "::notice title=Runner Selection::Using Warp Runners" else - echo "use-cirrus-runners=false" >> "$GITHUB_OUTPUT" + echo "use-warp-runners=false" >> "$GITHUB_OUTPUT" echo "::notice title=Runner Selection::Using GitHub-hosted runners" fi test-each-commit: name: 'test each commit' - runs-on: ubuntu-24.04 + needs: runners + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }} if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below. env: @@ -292,7 +292,7 @@ jobs: ci-matrix: name: ${{ matrix.name }} needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && matrix.cirrus-runner || matrix.fallback-runner }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && matrix.warp-runner || matrix.fallback-runner }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: ${{ matrix.timeout-minutes }} @@ -305,74 +305,74 @@ jobs: matrix: include: - name: '32 bit ARM, unit tests, no functional tests' - cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode + warp-runner: 'ubuntu-24.04-arm' # Warp's Arm runners don't support 32-bit mode currently fallback-runner: 'ubuntu-24.04-arm' timeout-minutes: 120 file-env: './ci/test/00_setup_env_arm.sh' - name: 'win64 Cross' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_win64.sh' - name: 'ASan + LSan + UBSan + integer, no depends, USDT' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-2404-x64-8x' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_asan.sh' - name: 'macOS-cross, gui, no tests' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross.sh' - name: 'No wallet, libbitcoinkernel' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh' - name: 'i686, multiprocess, DEBUG' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_i686_multiprocess.sh' - name: 'fuzzer,address,undefined,integer, no depends' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 240 file-env: './ci/test/00_setup_env_native_fuzz.sh' - name: 'previous releases, depends DEBUG' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_previous_releases.sh' - name: 'CentOS, depends, gui' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_centos.sh' - name: 'tidy' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tidy.sh' - name: 'TSan, depends, no gui' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tsan.sh' - name: 'MSan, depends' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_msan.sh' @@ -389,7 +389,7 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker with: - use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }} + use-warp: ${{ needs.runners.outputs.use-warp-runners }} - name: Enable bpfcc script if: ${{ env.CONTAINER_NAME == 'ci_native_asan' }} @@ -411,7 +411,7 @@ jobs: lint: name: 'lint' needs: runners - runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-2x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: @@ -426,7 +426,7 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker with: - use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }} + use-warp: ${{ needs.runners.outputs.use-warp-runners }} - name: CI script run: | diff --git a/ci/README.md b/ci/README.md index 81e048ce687..a73c8df5e75 100644 --- a/ci/README.md +++ b/ci/README.md @@ -61,13 +61,11 @@ trigger cache-invalidation and rebuilds as necessary. To configure the primary repository, follow these steps: -1. Register with [Cirrus Runners](https://cirrus-runners.app/) and purchase runners. -2. Install the Cirrus Runners GitHub app against the GitHub organization. +1. Register with [WarpBuild](https://www.warpbuild.com/) and purchase runners. +2. Install the WarpBuild GitHub app against the GitHub organization. 3. Enable organisation-level runners to be used in public repositories: 1. `Org settings -> Actions -> Runner Groups -> Default -> Allow public repos` 4. Permit the following actions to run: - 1. cirruslabs/cache/restore@\* - 1. cirruslabs/cache/save@\* 1. docker/setup-buildx-action@\* 1. actions/github-script@\* @@ -76,5 +74,5 @@ To configure the primary repository, follow these steps: When used in a fork the CI will run on GitHub's free hosted runners by default. In this case, due to GitHub's 10GB-per-repo cache size limitations caches will be frequently evicted and missed, but the workflows will run (slowly). -It is also possible to use your own Cirrus Runners in your own fork with an appropriate patch to the `REPO_USE_CIRRUS_RUNNERS` variable in ../.github/workflows/ci.yml -NB that Cirrus Runners only work at an organisation level, therefore in order to use your own Cirrus Runners, *the fork must be within your own organisation*. +It is also possible to use your own WarpBuild Runners in your own fork with an appropriate patch to the `REPO_USE_WARP_RUNNERS` variable in ../.github/workflows/ci.yml +NB that WarpBuild Runners only work at an organisation level, therefore in order to use your own WarpBuild Runners, *the fork must be within your own organisation*. diff --git a/depends/gen_id b/depends/gen_id index fe6d163547a..ab5b6fb58c6 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -28,6 +28,10 @@ # Redirect stderr to stdout exec 2>&1 + # Unset SOURCE_DATE_EPOCH to prevent it from leaking into tool + # outputs and to maximize reuse of the built package cache. + unset SOURCE_DATE_EPOCH + echo "BEGIN ALL" # Include any ID salts supplied via command line diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index 549d189e7d1..63318840119 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -86,7 +86,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` #### Notifications ###### ZeroMQ -Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +Bitcoin Core can provide notifications via ZeroMQ. To compile ZMQ support, install the following dependency and pass `-DWITH_ZMQ=ON` when configuring. ```bash pkg install libzmq4 ``` diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index 05b80eb7476..5b01d7b912c 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -86,7 +86,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` #### Notifications ###### ZeroMQ -Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +Bitcoin Core can provide notifications via ZeroMQ. To compile ZMQ support, install the following dependency and pass `-DWITH_ZMQ=ON` when configuring. ```bash pkgin zeromq ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 7ab51054ad2..e2a15f67c86 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -80,7 +80,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` #### Notifications ###### ZeroMQ -Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +Bitcoin Core can provide notifications via ZeroMQ. To compile ZMQ support, install the following dependency and pass `-DWITH_ZMQ=ON` when configuring. ```bash pkg_add zeromq ``` diff --git a/doc/release-notes.md b/doc/release-notes.md index c69faf32572..bd6db525f5d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -37,12 +37,25 @@ unsupported systems. Notable changes =============== +### Validation + +- #35209 validation: correct lifetime of precomputed tx data + +### Leveldb + +- #61(bitcoin-core/leveldb): Disable seek compaction + ### Net - #34093 netif: fix compilation warning in QueryDefaultGatewayImpl() +### Wallet + +- #35228 wallet: use outpoint when estimating input size + ### Build +- #34228 depends: Unset SOURCE_DATE_EPOCH in gen_id script - #34848 cmake: Migrate away from deprecated SQLite3 target ### Doc @@ -50,10 +63,13 @@ Notable changes - #34510 doc: fix broken bpftrace installation link - #34561 wallet: rpc: manpage: fix example missing `fee_rate` argument - #34671 doc: Update Guix install for Debian/Ubuntu +- #35283 doc: mention -DWITH_ZMQ=ON in BSD build guides ### CI - #35202 ci: restore sockets in i686, no IPC job +- #35378 ci: switch runners from cirrus to warpbuild +- #35408 ci: 35378 followups ### Misc @@ -64,14 +80,19 @@ Credits Thanks to everyone who directly contributed to this release: +- andrewtoth - Cory Fields - Daniel Pfeifer +- darosior +- fanquake - Hennadii Stepanov - jayvaliya +- junbyjun1238 - Lőrinc - MarcoFalke - SomberNight - ToRyVand +- willcl-ark As well as to everyone that helped with translations on [Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/src/leveldb/db/autocompact_test.cc b/src/leveldb/db/autocompact_test.cc index e6c97a05a6b..f6e714c6524 100644 --- a/src/leveldb/db/autocompact_test.cc +++ b/src/leveldb/db/autocompact_test.cc @@ -54,8 +54,8 @@ static const int kValueSize = 200 * 1024; static const int kTotalSize = 100 * 1024 * 1024; static const int kCount = kTotalSize / kValueSize; -// Read through the first n keys repeatedly and check that they get -// compacted (verified by checking the size of the key space). +// Read through the first n keys repeatedly and check that reads do NOT +// trigger compaction (seek compaction is disabled in this fork). void AutoCompactTest::DoReads(int n) { std::string value(kValueSize, 'x'); DBImpl* dbi = reinterpret_cast(db_); @@ -76,25 +76,23 @@ void AutoCompactTest::DoReads(int n) { const int64_t initial_size = Size(Key(0), Key(n)); const int64_t initial_other_size = Size(Key(n), Key(kCount)); - // Read until size drops significantly. + // Read repeatedly. The size of the read range must NOT shrink: with + // seek compaction disabled, reads never schedule a compaction. std::string limit_key = Key(n); - for (int read = 0; true; read++) { - ASSERT_LT(read, 100) << "Taking too long to compact"; + for (int read = 0; read < 100; read++) { Iterator* iter = db_->NewIterator(ReadOptions()); for (iter->SeekToFirst(); iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) { // Drop data } delete iter; - // Wait a little bit to allow any triggered compactions to complete. - Env::Default()->SleepForMicroseconds(1000000); uint64_t size = Size(Key(0), Key(n)); fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1, size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0); - if (size <= initial_size / 10) { - break; - } } + // Give any background work a chance to run, even though none should. + Env::Default()->SleepForMicroseconds(1000000); + ASSERT_EQ(Size(Key(0), Key(n)), static_cast(initial_size)); // Verify that the size of the key space not touched by the reads // is pretty much unchanged. diff --git a/src/leveldb/db/db_test.cc b/src/leveldb/db/db_test.cc index beb1d3bdef6..81fb9e94ebf 100644 --- a/src/leveldb/db/db_test.cc +++ b/src/leveldb/db/db_test.cc @@ -735,15 +735,14 @@ TEST(DBTest, GetPicksCorrectFile) { } while (ChangeOptions()); } -TEST(DBTest, GetEncountersEmptyLevel) { +TEST(DBTest, GetDoesNotTriggerSeekCompaction) { do { // Arrange for the following to happen: // * sstable A in level 0 // * nothing in level 1 // * sstable B in level 2 - // Then do enough Get() calls to arrange for an automatic compaction - // of sstable A. A bug would cause the compaction to be marked as - // occurring at level 1 (instead of the correct level 0). + // Seek compaction is disabled in this fork, so repeated reads must + // not change the level layout. A manual compaction must still work. // Step 1: First place sstables in levels 0 and 2 int compaction_count = 0; @@ -761,14 +760,17 @@ TEST(DBTest, GetEncountersEmptyLevel) { ASSERT_EQ(NumTableFilesAtLevel(1), 0); ASSERT_EQ(NumTableFilesAtLevel(2), 1); - // Step 3: read a bunch of times + // Step 3: many read misses must not schedule any compaction. for (int i = 0; i < 1000; i++) { ASSERT_EQ("NOT_FOUND", Get("missing")); } - - // Step 4: Wait for compaction to finish DelayMilliseconds(1000); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_EQ(NumTableFilesAtLevel(1), 0); + ASSERT_EQ(NumTableFilesAtLevel(2), 1); + // Step 4: a manual compaction still moves the L0 file down. + dbfull()->TEST_CompactRange(0, nullptr, nullptr); ASSERT_EQ(NumTableFilesAtLevel(0), 0); } while (ChangeOptions()); } diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index cd07346ea8a..8dc73295b84 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -400,16 +400,11 @@ Status Version::Get(const ReadOptions& options, const LookupKey& k, return state.found ? state.s : Status::NotFound(Slice()); } -bool Version::UpdateStats(const GetStats& stats) { - FileMetaData* f = stats.seek_file; - if (f != nullptr) { - f->allowed_seeks--; - if (f->allowed_seeks <= 0 && file_to_compact_ == nullptr) { - file_to_compact_ = f; - file_to_compact_level_ = stats.seek_file_level; - return true; - } - } +bool Version::UpdateStats(const GetStats& /*stats*/) { + // Disable automatic compactions triggered by read seek counters. + // The heuristic was tuned for expensive random seeks and can create + // severe write amplification on large random-key databases. + // Size and manual compactions still run. return false; } @@ -661,6 +656,8 @@ class VersionSet::Builder { // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. + // + // Note: seek compactions are disabled. See Version::UpdateStats. f->allowed_seeks = static_cast((f->file_size / 16384U)); if (f->allowed_seeks < 100) f->allowed_seeks = 100; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index f45db1c16f3..26924d63526 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -92,7 +92,7 @@ int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoin if (!provider) return -1; if (const auto desc = InferDescriptor(txout.scriptPubKey, *provider)) { - if (const auto weight = MaxInputWeight(*desc, {}, coin_control, true, can_grind_r)) { + if (const auto weight = MaxInputWeight(*desc, CTxIn{outpoint}, coin_control, true, can_grind_r)) { return static_cast(GetVirtualTransactionSize(*weight, 0, 0)); } } diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index 963c0f838b1..ac76fe4dda0 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include