mirror of
https://github.com/bitcoin/bitcoin.git
synced 2025-05-05 17:30:59 +02:00
2f71a1ea35
f19fdd47a6371dcbe0760ef6f3c3c5adb31b1bb4 test: add test for CChainState::ResizeCoinsCaches() (James O'Beirne) 8ac3ef46999ed676ca3775f7b2f461d92f09a542 add ChainstateManager::MaybeRebalanceCaches() (James O'Beirne) f36aaa6392fdbdac6891d92202d3efeff98754f4 Add CChainState::ResizeCoinsCaches (James O'Beirne) b223111da2e0e9ceccef75df8a20252b0094b7bc txdb: add CCoinsViewDB::ChangeCacheSize (James O'Beirne) Pull request description: This is part of the [assumeutxo project](https://github.com/bitcoin/bitcoin/projects/11): Parent PR: #15606 Issue: #15605 Specification: https://github.com/jamesob/assumeutxo-docs/tree/master/proposal --- In the assumeutxo implementation draft (#15056), once a UTXO snapshot is loaded, a new chainstate object is created after initialization. This means that we have to reclaim some of the cache that we've allocated to the original chainstate (per `dbcache=`) to repurpose for the snapshot chainstate. Furthermore, it makes sense to have different cache allocations depending on which chainstate is more active. While the snapshot chainstate is working to get to the network tip (and the background validation chainstate is idle), it makes sense that the snapshot chainstate should have the majority of cache allocation. And contrariwise once the snapshot has reached network tip, most of the cache should be given to the background validation chainstate. This set of changes (detailed in the commit messages) allows us to dynamically resize the various coins caches. None of the functionality introduced here is used at the moment, but will be in the next AU PR (which introduces `ActivateSnapshot`). `ChainstateManager::MaybeRebalanceCaches()` defines the (somewhat normative) cache allocations between the snapshot and background validation chainstates. I'd be interested in feedback if anyone has thoughts on the proportions I've set there. ACKs for top commit: ajtowns: weak utACK f19fdd47a6371dcbe0760ef6f3c3c5adb31b1bb4 -- didn't find any major problems, but not super confident that I didn't miss anything fjahr: Code review ACK f19fdd4 ryanofsky: Code review ACK f19fdd47a6371dcbe0760ef6f3c3c5adb31b1bb4. Only change since last review is constructor cleanup (no change in behavior). I think the suggestions here from ajtowns and others are good, but shouldn't delay merging the PR (and hold up assumeutxo) Tree-SHA512: fffb7847fb6993dd4a1a41cf11179b211b0b20b7eb5f7cf6266442136bfe9d43b830bbefcafd475bfd4af273f5573500594aa41fff03e0ed5c2a1e8562ff9269
267 lines
15 KiB
C++
267 lines
15 KiB
C++
// Copyright (c) 2011-2020 The Bitcoin Core developers
|
|
// Distributed under the MIT software license, see the accompanying
|
|
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
|
|
|
#include <test/util/setup_common.h>
|
|
|
|
#include <banman.h>
|
|
#include <chainparams.h>
|
|
#include <consensus/consensus.h>
|
|
#include <consensus/params.h>
|
|
#include <consensus/validation.h>
|
|
#include <crypto/sha256.h>
|
|
#include <init.h>
|
|
#include <miner.h>
|
|
#include <net.h>
|
|
#include <net_processing.h>
|
|
#include <noui.h>
|
|
#include <pow.h>
|
|
#include <rpc/blockchain.h>
|
|
#include <rpc/register.h>
|
|
#include <rpc/server.h>
|
|
#include <scheduler.h>
|
|
#include <script/sigcache.h>
|
|
#include <streams.h>
|
|
#include <txdb.h>
|
|
#include <util/memory.h>
|
|
#include <util/strencodings.h>
|
|
#include <util/string.h>
|
|
#include <util/time.h>
|
|
#include <util/translation.h>
|
|
#include <util/url.h>
|
|
#include <util/vector.h>
|
|
#include <validation.h>
|
|
#include <validationinterface.h>
|
|
|
|
#include <functional>
|
|
|
|
const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr;
|
|
UrlDecodeFn* const URL_DECODE = nullptr;
|
|
|
|
FastRandomContext g_insecure_rand_ctx;
|
|
/** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */
|
|
static FastRandomContext g_insecure_rand_ctx_temp_path;
|
|
|
|
/** Return the unsigned from the environment var if available, otherwise 0 */
|
|
static uint256 GetUintFromEnv(const std::string& env_name)
|
|
{
|
|
const char* num = std::getenv(env_name.c_str());
|
|
if (!num) return {};
|
|
return uint256S(num);
|
|
}
|
|
|
|
void Seed(FastRandomContext& ctx)
|
|
{
|
|
// Should be enough to get the seed once for the process
|
|
static uint256 seed{};
|
|
static const std::string RANDOM_CTX_SEED{"RANDOM_CTX_SEED"};
|
|
if (seed.IsNull()) seed = GetUintFromEnv(RANDOM_CTX_SEED);
|
|
if (seed.IsNull()) seed = GetRandHash();
|
|
LogPrintf("%s: Setting random seed for current tests to %s=%s\n", __func__, RANDOM_CTX_SEED, seed.GetHex());
|
|
ctx = FastRandomContext(seed);
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& os, const uint256& num)
|
|
{
|
|
os << num.ToString();
|
|
return os;
|
|
}
|
|
|
|
BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args)
|
|
: m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()}
|
|
{
|
|
const std::vector<const char*> arguments = Cat(
|
|
{
|
|
"dummy",
|
|
"-printtoconsole=0",
|
|
"-logtimemicros",
|
|
"-logthreadnames",
|
|
"-debug",
|
|
"-debugexclude=libevent",
|
|
"-debugexclude=leveldb",
|
|
},
|
|
extra_args);
|
|
util::ThreadRename("test");
|
|
fs::create_directories(m_path_root);
|
|
gArgs.ForceSetArg("-datadir", m_path_root.string());
|
|
ClearDatadirCache();
|
|
{
|
|
SetupServerArgs(m_node);
|
|
std::string error;
|
|
const bool success{m_node.args->ParseParameters(arguments.size(), arguments.data(), error)};
|
|
assert(success);
|
|
assert(error.empty());
|
|
}
|
|
SelectParams(chainName);
|
|
SeedInsecureRand();
|
|
if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN);
|
|
InitLogging();
|
|
AppInitParameterInteraction();
|
|
LogInstance().StartLogging();
|
|
SHA256AutoDetect();
|
|
ECC_Start();
|
|
SetupEnvironment();
|
|
SetupNetworking();
|
|
InitSignatureCache();
|
|
InitScriptExecutionCache();
|
|
fCheckBlockIndex = true;
|
|
static bool noui_connected = false;
|
|
if (!noui_connected) {
|
|
noui_connect();
|
|
noui_connected = true;
|
|
}
|
|
}
|
|
|
|
BasicTestingSetup::~BasicTestingSetup()
|
|
{
|
|
LogInstance().DisconnectTestLogger();
|
|
fs::remove_all(m_path_root);
|
|
gArgs.ClearArgs();
|
|
ECC_Stop();
|
|
}
|
|
|
|
TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args)
|
|
: BasicTestingSetup(chainName, extra_args)
|
|
{
|
|
const CChainParams& chainparams = Params();
|
|
// Ideally we'd move all the RPC tests to the functional testing framework
|
|
// instead of unit tests, but for now we need these here.
|
|
RegisterAllCoreRPCCommands(tableRPC);
|
|
|
|
m_node.scheduler = MakeUnique<CScheduler>();
|
|
|
|
// We have to run a scheduler thread to prevent ActivateBestChain
|
|
// from blocking due to queue overrun.
|
|
threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); });
|
|
GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler);
|
|
|
|
pblocktree.reset(new CBlockTreeDB(1 << 20, true));
|
|
|
|
m_node.chainman = &::g_chainman;
|
|
m_node.chainman->InitializeChainstate();
|
|
::ChainstateActive().InitCoinsDB(
|
|
/* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false);
|
|
assert(!::ChainstateActive().CanFlushToDisk());
|
|
::ChainstateActive().InitCoinsCache(1 << 23);
|
|
assert(::ChainstateActive().CanFlushToDisk());
|
|
if (!LoadGenesisBlock(chainparams)) {
|
|
throw std::runtime_error("LoadGenesisBlock failed.");
|
|
}
|
|
|
|
BlockValidationState state;
|
|
if (!ActivateBestChain(state, chainparams)) {
|
|
throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString()));
|
|
}
|
|
|
|
// Start script-checking threads. Set g_parallel_script_checks to true so they are used.
|
|
constexpr int script_check_threads = 2;
|
|
for (int i = 0; i < script_check_threads; ++i) {
|
|
threadGroup.create_thread([i]() { return ThreadScriptCheck(i); });
|
|
}
|
|
g_parallel_script_checks = true;
|
|
|
|
m_node.mempool = &::mempool;
|
|
m_node.mempool->setSanityCheck(1.0);
|
|
m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME);
|
|
m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests.
|
|
m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool);
|
|
{
|
|
CConnman::Options options;
|
|
options.m_msgproc = m_node.peer_logic.get();
|
|
m_node.connman->Init(options);
|
|
}
|
|
}
|
|
|
|
TestingSetup::~TestingSetup()
|
|
{
|
|
if (m_node.scheduler) m_node.scheduler->stop();
|
|
threadGroup.interrupt_all();
|
|
threadGroup.join_all();
|
|
GetMainSignals().FlushBackgroundCallbacks();
|
|
GetMainSignals().UnregisterBackgroundSignalScheduler();
|
|
m_node.connman.reset();
|
|
m_node.banman.reset();
|
|
m_node.args = nullptr;
|
|
m_node.mempool = nullptr;
|
|
m_node.scheduler.reset();
|
|
UnloadBlockIndex();
|
|
m_node.chainman->Reset();
|
|
m_node.chainman = nullptr;
|
|
pblocktree.reset();
|
|
}
|
|
|
|
TestChain100Setup::TestChain100Setup()
|
|
{
|
|
// CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests.
|
|
// TODO: fix the code to support SegWit blocks.
|
|
gArgs.ForceSetArg("-segwitheight", "432");
|
|
// Need to recreate chainparams
|
|
SelectParams(CBaseChainParams::REGTEST);
|
|
|
|
// Generate a 100-block chain:
|
|
coinbaseKey.MakeNewKey(true);
|
|
CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG;
|
|
for (int i = 0; i < COINBASE_MATURITY; i++)
|
|
{
|
|
std::vector<CMutableTransaction> noTxns;
|
|
CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey);
|
|
m_coinbase_txns.push_back(b.vtx[0]);
|
|
}
|
|
}
|
|
|
|
// Create a new block with just given transactions, coinbase paying to
|
|
// scriptPubKey, and try to add it to the current chain.
|
|
CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey)
|
|
{
|
|
const CChainParams& chainparams = Params();
|
|
std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey);
|
|
CBlock& block = pblocktemplate->block;
|
|
|
|
// Replace mempool-selected txns with just coinbase plus passed-in txns:
|
|
block.vtx.resize(1);
|
|
for (const CMutableTransaction& tx : txns)
|
|
block.vtx.push_back(MakeTransactionRef(tx));
|
|
// IncrementExtraNonce creates a valid coinbase and merkleRoot
|
|
{
|
|
LOCK(cs_main);
|
|
unsigned int extraNonce = 0;
|
|
IncrementExtraNonce(&block, ::ChainActive().Tip(), extraNonce);
|
|
}
|
|
|
|
while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce;
|
|
|
|
std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block);
|
|
Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr);
|
|
|
|
CBlock result = block;
|
|
return result;
|
|
}
|
|
|
|
TestChain100Setup::~TestChain100Setup()
|
|
{
|
|
gArgs.ForceSetArg("-segwitheight", "0");
|
|
}
|
|
|
|
|
|
CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx) {
|
|
return FromTx(MakeTransactionRef(tx));
|
|
}
|
|
|
|
CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx)
|
|
{
|
|
return CTxMemPoolEntry(tx, nFee, nTime, nHeight,
|
|
spendsCoinbase, sigOpCost, lp);
|
|
}
|
|
|
|
/**
|
|
* @returns a real block (0000000000013b8ab2cd513b0261a14096412195a72a0c4827d229dcc7e0f7af)
|
|
* with 9 txs.
|
|
*/
|
|
CBlock getBlock13b8a()
|
|
{
|
|
CBlock block;
|
|
CDataStream stream(ParseHex("0100000090f0a9f110702f808219ebea1173056042a714bad51b916cb6800000000000005275289558f51c9966699404ae2294730c3c9f9bda53523ce50e9b95e558da2fdb261b4d4c86041b1ab1bf930901000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0146ffffffff0100f2052a01000000434104e18f7afbe4721580e81e8414fc8c24d7cfacf254bb5c7b949450c3e997c2dc1242487a8169507b631eb3771f2b425483fb13102c4eb5d858eef260fe70fbfae0ac00000000010000000196608ccbafa16abada902780da4dc35dafd7af05fa0da08cf833575f8cf9e836000000004a493046022100dab24889213caf43ae6adc41cf1c9396c08240c199f5225acf45416330fd7dbd022100fe37900e0644bf574493a07fc5edba06dbc07c311b947520c2d514bc5725dcb401ffffffff0100f2052a010000001976a914f15d1921f52e4007b146dfa60f369ed2fc393ce288ac000000000100000001fb766c1288458c2bafcfec81e48b24d98ec706de6b8af7c4e3c29419bfacb56d000000008c493046022100f268ba165ce0ad2e6d93f089cfcd3785de5c963bb5ea6b8c1b23f1ce3e517b9f022100da7c0f21adc6c401887f2bfd1922f11d76159cbc597fbd756a23dcbb00f4d7290141042b4e8625a96127826915a5b109852636ad0da753c9e1d5606a50480cd0c40f1f8b8d898235e571fe9357d9ec842bc4bba1827daaf4de06d71844d0057707966affffffff0280969800000000001976a9146963907531db72d0ed1a0cfb471ccb63923446f388ac80d6e34c000000001976a914f0688ba1c0d1ce182c7af6741e02658c7d4dfcd388ac000000000100000002c40297f730dd7b5a99567eb8d27b78758f607507c52292d02d4031895b52f2ff010000008b483045022100f7edfd4b0aac404e5bab4fd3889e0c6c41aa8d0e6fa122316f68eddd0a65013902205b09cc8b2d56e1cd1f7f2fafd60a129ed94504c4ac7bdc67b56fe67512658b3e014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffffca5065ff9617cbcba45eb23726df6498a9b9cafed4f54cbab9d227b0035ddefb000000008a473044022068010362a13c7f9919fa832b2dee4e788f61f6f5d344a7c2a0da6ae740605658022006d1af525b9a14a35c003b78b72bd59738cd676f845d1ff3fc25049e01003614014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffff01001ec4110200000043410469ab4181eceb28985b9b4e895c13fa5e68d85761b7eee311db5addef76fa8621865134a221bd01f28ec9999ee3e021e60766e9d1f3458c115fb28650605f11c9ac000000000100000001cdaf2f758e91c514655e2dc50633d1e4c84989f8aa90a0dbc883f0d23ed5c2fa010000008b48304502207ab51be6f12a1962ba0aaaf24a20e0b69b27a94fac5adf45aa7d2d18ffd9236102210086ae728b370e5329eead9accd880d0cb070aea0c96255fae6c4f1ddcce1fd56e014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff02404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac002d3101000000001976a9141befba0cdc1ad56529371864d9f6cb042faa06b588ac000000000100000001b4a47603e71b61bc3326efd90111bf02d2f549b067f4c4a8fa183b57a0f800cb010000008a4730440220177c37f9a505c3f1a1f0ce2da777c339bd8339ffa02c7cb41f0a5804f473c9230220585b25a2ee80eb59292e52b987dad92acb0c64eced92ed9ee105ad153cdb12d001410443bd44f683467e549dae7d20d1d79cbdb6df985c6e9c029c8d0c6cb46cc1a4d3cf7923c5021b27f7a0b562ada113bc85d5fda5a1b41e87fe6e8802817cf69996ffffffff0280651406000000001976a9145505614859643ab7b547cd7f1f5e7e2a12322d3788ac00aa0271000000001976a914ea4720a7a52fc166c55ff2298e07baf70ae67e1b88ac00000000010000000586c62cd602d219bb60edb14a3e204de0705176f9022fe49a538054fb14abb49e010000008c493046022100f2bc2aba2534becbdf062eb993853a42bbbc282083d0daf9b4b585bd401aa8c9022100b1d7fd7ee0b95600db8535bbf331b19eed8d961f7a8e54159c53675d5f69df8c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff03ad0e58ccdac3df9dc28a218bcf6f1997b0a93306faaa4b3a28ae83447b2179010000008b483045022100be12b2937179da88599e27bb31c3525097a07cdb52422d165b3ca2f2020ffcf702200971b51f853a53d644ebae9ec8f3512e442b1bcb6c315a5b491d119d10624c83014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff2acfcab629bbc8685792603762c921580030ba144af553d271716a95089e107b010000008b483045022100fa579a840ac258871365dd48cd7552f96c8eea69bd00d84f05b283a0dab311e102207e3c0ee9234814cfbb1b659b83671618f45abc1326b9edcc77d552a4f2a805c0014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffdcdc6023bbc9944a658ddc588e61eacb737ddf0a3cd24f113b5a8634c517fcd2000000008b4830450221008d6df731df5d32267954bd7d2dda2302b74c6c2a6aa5c0ca64ecbabc1af03c75022010e55c571d65da7701ae2da1956c442df81bbf076cdbac25133f99d98a9ed34c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffe15557cd5ce258f479dfd6dc6514edf6d7ed5b21fcfa4a038fd69f06b83ac76e010000008b483045022023b3e0ab071eb11de2eb1cc3a67261b866f86bf6867d4558165f7c8c8aca2d86022100dc6e1f53a91de3efe8f63512850811f26284b62f850c70ca73ed5de8771fb451014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff01404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000010000000166d7577163c932b4f9690ca6a80b6e4eb001f0a2fa9023df5595602aae96ed8d000000008a4730440220262b42546302dfb654a229cefc86432b89628ff259dc87edd1154535b16a67e102207b4634c020a97c3e7bbd0d4d19da6aa2269ad9dded4026e896b213d73ca4b63f014104979b82d02226b3a4597523845754d44f13639e3bf2df5e82c6aab2bdc79687368b01b1ab8b19875ae3c90d661a3d0a33161dab29934edeb36aa01976be3baf8affffffff02404b4c00000000001976a9144854e695a02af0aeacb823ccbc272134561e0a1688ac40420f00000000001976a914abee93376d6b37b5c2940655a6fcaf1c8e74237988ac0000000001000000014e3f8ef2e91349a9059cb4f01e54ab2597c1387161d3da89919f7ea6acdbb371010000008c49304602210081f3183471a5ca22307c0800226f3ef9c353069e0773ac76bb580654d56aa523022100d4c56465bdc069060846f4fbf2f6b20520b2a80b08b168b31e66ddb9c694e240014104976c79848e18251612f8940875b2b08d06e6dc73b9840e8860c066b7e87432c477e9a59a453e71e6d76d5fe34058b800a098fc1740ce3012e8fc8a00c96af966ffffffff02c0e1e400000000001976a9144134e75a6fcb6042034aab5e18570cf1f844f54788ac404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000"), SER_NETWORK, PROTOCOL_VERSION);
|
|
stream >> block;
|
|
return block;
|
|
}
|