highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-11-11 06:28:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge bitcoin/bitcoin#25667: assumeutxo: snapshot initialization

Browse Source 
bf95976061 doc: add note about snapshot chainstate init (James O'Beirne)
e4d7995286 test: add testcases for snapshot initialization (James O'Beirne)
cced4e7336 test: move-only-ish: factor out LoadVerifyActivateChainstate() (James O'Beirne)
51fc9241c0 test: allow on-disk coins and block tree dbs in tests (James O'Beirne)
3c361391b8 test: add reset_chainstate parameter for snapshot unittests (James O'Beirne)
00b357c215 validation: add ResetChainstates() (James O'Beirne)
3a29dfbfb2 move-only: test: make snapshot chainstate setup reusable (James O'Beirne)
8153bd9247 blockmanager: avoid undefined behavior during FlushBlockFile (James O'Beirne)
ad67ff377c validation: remove snapshot datadirs upon validation failure (James O'Beirne)
34d1590331 add utilities for deleting on-disk leveldb data (James O'Beirne)
252abd1e8b init: add utxo snapshot detection (James O'Beirne)
f9f1735f13 validation: rename snapshot chainstate dir (James O'Beirne)
d14bebf100 db: add StoragePath to CDBWrapper/CCoinsViewDB (James O'Beirne)

Pull request description:

  This is part of the [assumeutxo project](https://github.com/bitcoin/bitcoin/projects/11) (parent PR: https://github.com/bitcoin/bitcoin/pull/15606)

  ---

  Half of the replacement for #24232. The original PR grew larger than expected throughout the review process.

  This change adds the ability to initialize a snapshot-based chainstate during init if one is detected on disk. This is of course unused as of now (aside from in unittests) given that we haven't yet enabled actually loading snapshots.

  Don't be scared! There are some big move-only commits in here.

  Accompanying changes include:

  - moving the snapshot coinsdb directory from being called `chainstate_[base blockhash]` to `chainstate_snapshot`, since we only support one snapshot in use at a time. This simplifies some logic, but it necessitates writing that base blockhash out to a file within the coinsdb dir. See [discussion here](https://github.com/bitcoin/bitcoin/pull/24232#discussion_r832762880).
  - adding a simple fix in `FlushBlockFile()` that avoids a crash when attemping to flush to disk before `LoadBlockIndexDB()` is called, which happens when calling `MaybeRebalanceCaches()` during multiple chainstate init.
  - improving the unittest to allow testing with on-disk chainstates - necessary to test a simulated restart and re-initialization.

ACKs for top commit:
  naumenkogs:
    utACK bf95976061
  ariard:
    Code Review ACK bf9597606
  ryanofsky:
    Code review ACK bf95976061. Changes since last review: rebasing, switching from CAutoFile to AutoFile, adding comments, switching from BOOST_CHECK to Assert in test util, using chainman.GetMutex() in tests, destroying one ChainstateManager before creating a new one in tests
  fjahr:
    utACK bf95976061
  aureleoules:
    ACK bf95976061

Tree-SHA512: 15ae75caf19f8d12a12d2647c52897904d27b265a7af6b4ae7b858592eeadb8f9da6c2394b6baebec90adc28742c053e3eb506119577dae7c1e722ebb3b7bcc0
This commit is contained in:
Andrew Chow
2022-10-13 10:08:01 -04:00
parent 147d64dbdf bf95976061
commit 6912a28f08
19 changed files with 662 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

396
src/test/validation_chainstatemanager_tests.cpp
View File

@@ -10,6 +10,7 @@
#include <sync.h>
#include <test/util/chainstate.h>
#include <test/util/setup_common.h>
#include <timedata.h>
#include <uint256.h>
#include <validation.h>
#include <validationinterface.h>
@@ -63,7 +64,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager)
// Create a snapshot-based chainstate.
//
const uint256 snapshot_blockhash = GetRandHash();
Chainstate& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(
Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot(
&mempool, snapshot_blockhash));
chainstates.push_back(&c2);
@@ -133,7 +134,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches)
// Create a snapshot-based chainstate.
//
Chainstate& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash()));
Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash()));
chainstates.push_back(&c2);
c2.InitCoinsDB(
/*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false);
@@ -154,162 +155,240 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches)
BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1);
}
//! Test basic snapshot activation.
BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup)
{
ChainstateManager& chainman = *Assert(m_node.chainman);
size_t initial_size;
size_t initial_total_coins{100};
// Make some initial assertions about the contents of the chainstate.
struct SnapshotTestSetup : TestChain100Setup {
// Run with coinsdb on the filesystem to support, e.g., moving invalidated
// chainstate dirs to "*_invalid".
//
// Note that this means the tests run considerably slower than in-memory DB
// tests, but we can't otherwise test this functionality since it relies on
// destructive filesystem operations.
SnapshotTestSetup() : TestChain100Setup{
{},
{},
/*coins_db_in_memory=*/false,
/*block_tree_db_in_memory=*/false,
}
{
LOCK(::cs_main);
CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip();
initial_size = ibd_coinscache.GetCacheSize();
size_t total_coins{0};
for (CTransactionRef& txn : m_coinbase_txns) {
COutPoint op{txn->GetHash(), 0};
BOOST_CHECK(ibd_coinscache.HaveCoin(op));
total_coins++;
}
BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
BOOST_CHECK_EQUAL(initial_size, initial_total_coins);
}
// Snapshot should refuse to load at this height.
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root));
BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash);
BOOST_CHECK(!chainman.SnapshotBlockhash());
// Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can
// be found.
constexpr int snapshot_height = 110;
mineBlocks(10);
initial_size += 10;
initial_total_coins += 10;
// Should not load malleated snapshots
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// A UTXO is missing but count is correct
metadata.m_coins_count -= 1;
COutPoint outpoint;
Coin coin;
auto_infile >> outpoint;
auto_infile >> coin;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Coins count is larger than coins in file
metadata.m_coins_count += 1;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Coins count is smaller than coins in file
metadata.m_coins_count -= 1;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Wrong hash
metadata.m_base_blockhash = uint256::ZERO;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Wrong hash
metadata.m_base_blockhash = uint256::ONE;
}));
BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root));
// Ensure our active chain is the snapshot chainstate.
BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull());
BOOST_CHECK_EQUAL(
*chainman.ActiveChainstate().m_from_snapshot_blockhash,
*chainman.SnapshotBlockhash());
// Ensure that the genesis block was not marked assumed-valid.
BOOST_CHECK(WITH_LOCK(::cs_main, return !chainman.ActiveChain().Genesis()->IsAssumedValid()));
const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params());
const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip());
BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx);
// To be checked against later when we try loading a subsequent snapshot.
uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()};
// Make some assertions about the both chainstates. These checks ensure the
// legacy chainstate hasn't changed and that the newly created chainstate
// reflects the expected content.
std::tuple<Chainstate*, Chainstate*> SetupSnapshot()
{
LOCK(::cs_main);
int chains_tested{0};
ChainstateManager& chainman = *Assert(m_node.chainman);
for (Chainstate* chainstate : chainman.GetAll()) {
BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
CCoinsViewCache& coinscache = chainstate->CoinsTip();
BOOST_CHECK(!chainman.IsSnapshotActive());
// Both caches will be empty initially.
BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize());
{
LOCK(::cs_main);
BOOST_CHECK(!chainman.IsSnapshotValidated());
BOOST_CHECK(!node::FindSnapshotChainstateDir());
}
size_t initial_size;
size_t initial_total_coins{100};
// Make some initial assertions about the contents of the chainstate.
{
LOCK(::cs_main);
CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip();
initial_size = ibd_coinscache.GetCacheSize();
size_t total_coins{0};
for (CTransactionRef& txn : m_coinbase_txns) {
COutPoint op{txn->GetHash(), 0};
BOOST_CHECK(coinscache.HaveCoin(op));
BOOST_CHECK(ibd_coinscache.HaveCoin(op));
total_coins++;
}
BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize());
BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
chains_tested++;
BOOST_CHECK_EQUAL(initial_size, initial_total_coins);
}
BOOST_CHECK_EQUAL(chains_tested, 2);
}
Chainstate& validation_chainstate = chainman.ActiveChainstate();
// Mine some new blocks on top of the activated snapshot chainstate.
constexpr size_t new_coins{100};
mineBlocks(new_coins); // Defined in TestChain100Setup.
// Snapshot should refuse to load at this height.
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this));
BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash);
BOOST_CHECK(!chainman.SnapshotBlockhash());
{
LOCK(::cs_main);
size_t coins_in_active{0};
size_t coins_in_background{0};
size_t coins_missing_from_background{0};
// Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can
// be found.
constexpr int snapshot_height = 110;
mineBlocks(10);
initial_size += 10;
initial_total_coins += 10;
for (Chainstate* chainstate : chainman.GetAll()) {
BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
CCoinsViewCache& coinscache = chainstate->CoinsTip();
bool is_background = chainstate != &chainman.ActiveChainstate();
// Should not load malleated snapshots
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// A UTXO is missing but count is correct
metadata.m_coins_count -= 1;
for (CTransactionRef& txn : m_coinbase_txns) {
COutPoint op{txn->GetHash(), 0};
if (coinscache.HaveCoin(op)) {
(is_background ? coins_in_background : coins_in_active)++;
} else if (is_background) {
coins_missing_from_background++;
COutPoint outpoint;
Coin coin;
auto_infile >> outpoint;
auto_infile >> coin;
}));
BOOST_CHECK(!node::FindSnapshotChainstateDir());
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Coins count is larger than coins in file
metadata.m_coins_count += 1;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Coins count is smaller than coins in file
metadata.m_coins_count -= 1;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Wrong hash
metadata.m_base_blockhash = uint256::ZERO;
}));
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(
this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) {
// Wrong hash
metadata.m_base_blockhash = uint256::ONE;
}));
BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this));
BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir()));
// Ensure our active chain is the snapshot chainstate.
BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull());
BOOST_CHECK_EQUAL(
*chainman.ActiveChainstate().m_from_snapshot_blockhash,
*chainman.SnapshotBlockhash());
Chainstate& snapshot_chainstate = chainman.ActiveChainstate();
{
LOCK(::cs_main);
fs::path found = *node::FindSnapshotChainstateDir();
// Note: WriteSnapshotBaseBlockhash() is implicitly tested above.
BOOST_CHECK_EQUAL(
*node::ReadSnapshotBaseBlockhash(found),
*chainman.SnapshotBlockhash());
// Ensure that the genesis block was not marked assumed-valid.
BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid());
}
const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params());
const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip());
BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx);
// To be checked against later when we try loading a subsequent snapshot.
uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()};
// Make some assertions about the both chainstates. These checks ensure the
// legacy chainstate hasn't changed and that the newly created chainstate
// reflects the expected content.
{
LOCK(::cs_main);
int chains_tested{0};
for (Chainstate* chainstate : chainman.GetAll()) {
BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
CCoinsViewCache& coinscache = chainstate->CoinsTip();
// Both caches will be empty initially.
BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize());
size_t total_coins{0};
for (CTransactionRef& txn : m_coinbase_txns) {
COutPoint op{txn->GetHash(), 0};
BOOST_CHECK(coinscache.HaveCoin(op));
total_coins++;
}
BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize());
BOOST_CHECK_EQUAL(total_coins, initial_total_coins);
chains_tested++;
}
BOOST_CHECK_EQUAL(chains_tested, 2);
}
// Mine some new blocks on top of the activated snapshot chainstate.
constexpr size_t new_coins{100};
mineBlocks(new_coins); // Defined in TestChain100Setup.
{
LOCK(::cs_main);
size_t coins_in_active{0};
size_t coins_in_background{0};
size_t coins_missing_from_background{0};
for (Chainstate* chainstate : chainman.GetAll()) {
BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString());
CCoinsViewCache& coinscache = chainstate->CoinsTip();
bool is_background = chainstate != &chainman.ActiveChainstate();
for (CTransactionRef& txn : m_coinbase_txns) {
COutPoint op{txn->GetHash(), 0};
if (coinscache.HaveCoin(op)) {
(is_background ? coins_in_background : coins_in_active)++;
} else if (is_background) {
coins_missing_from_background++;
}
}
}
BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins);
BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins);
BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins);
}
BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins);
BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins);
BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins);
// Snapshot should refuse to load after one has already loaded.
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this));
// Snapshot blockhash should be unchanged.
BOOST_CHECK_EQUAL(
*chainman.ActiveChainstate().m_from_snapshot_blockhash,
loaded_snapshot_blockhash);
return std::make_tuple(&validation_chainstate, &snapshot_chainstate);
}
// Snapshot should refuse to load after one has already loaded.
BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root));
// Simulate a restart of the node by flushing all state to disk, clearing the
// existing ChainstateManager, and unloading the block index.
//
// @returns a reference to the "restarted" ChainstateManager
ChainstateManager& SimulateNodeRestart()
{
ChainstateManager& chainman = *Assert(m_node.chainman);
// Snapshot blockhash should be unchanged.
BOOST_CHECK_EQUAL(
*chainman.ActiveChainstate().m_from_snapshot_blockhash,
loaded_snapshot_blockhash);
BOOST_TEST_MESSAGE("Simulating node restart");
{
LOCK(::cs_main);
for (Chainstate* cs : chainman.GetAll()) {
cs->ForceFlushStateToDisk();
}
chainman.ResetChainstates();
BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0);
const ChainstateManager::Options chainman_opts{
.chainparams = ::Params(),
.adjusted_time_callback = GetAdjustedTime,
};
// For robustness, ensure the old manager is destroyed before creating a
// new one.
m_node.chainman.reset();
m_node.chainman.reset(new ChainstateManager(chainman_opts));
}
return *Assert(m_node.chainman);
}
};
//! Test basic snapshot activation.
BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup)
{
this->SetupSnapshot();
}
//! Test LoadBlockIndex behavior when multiple chainstates are in use.
@@ -374,7 +453,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup)
BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid);
Chainstate& cs2 = WITH_LOCK(::cs_main,
return chainman.InitializeChainstate(&mempool, GetRandHash()));
return chainman.ActivateExistingSnapshot(&mempool, GetRandHash()));
reload_all_block_indexes();
@@ -390,4 +469,59 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup)
BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes);
}
//! Ensure that snapshot chainstates initialize properly when found on disk.
BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup)
{
this->SetupSnapshot();
ChainstateManager& chainman = *Assert(m_node.chainman);
fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir();
BOOST_CHECK(fs::exists(snapshot_chainstate_dir));
BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot");
BOOST_CHECK(chainman.IsSnapshotActive());
const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(),
return chainman.ActiveTip()->GetBlockHash());
auto all_chainstates = chainman.GetAll();
BOOST_CHECK_EQUAL(all_chainstates.size(), 2);
// Test that simulating a shutdown (resetting ChainstateManager) and then performing
// chainstate reinitializing successfully cleans up the background-validation
// chainstate data, and we end up with a single chainstate that is at tip.
ChainstateManager& chainman_restarted = this->SimulateNodeRestart();
BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate");
// This call reinitializes the chainstates.
this->LoadVerifyActivateChainstate();
{
LOCK(chainman_restarted.GetMutex());
BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 2);
BOOST_CHECK(chainman_restarted.IsSnapshotActive());
BOOST_CHECK(!chainman_restarted.IsSnapshotValidated());
BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash);
BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210);
}
BOOST_TEST_MESSAGE(
"Ensure we can mine blocks on top of the initialized snapshot chainstate");
mineBlocks(10);
{
LOCK(chainman_restarted.GetMutex());
BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220);
// Background chainstate should be unaware of new blocks on the snapshot
// chainstate.
for (Chainstate* cs : chainman_restarted.GetAll()) {
if (cs != &chainman_restarted.ActiveChainstate()) {
BOOST_CHECK_EQUAL(cs->m_chain.Height(), 110);
}
}
}
}
BOOST_AUTO_TEST_SUITE_END()