node/caches: Extract cache calculation logic

I strongly recommend reviewing with the following git-diff flags:
  --color-moved=dimmed_zebra --color-moved-ws=allow-indentation-change

[META] In a future commit, this function will be re-used in TestingSetup
       so that the behaviour matches across test and non-test init
       codepaths.
This commit is contained in:
Carl Dong
2021-09-21 11:37:03 -04:00
parent 15f2e33bb3
commit ac4bf138b8
4 changed files with 75 additions and 28 deletions

View File

@@ -174,6 +174,7 @@ BITCOIN_CORE_H = \
netbase.h \ netbase.h \
netmessagemaker.h \ netmessagemaker.h \
node/blockstorage.h \ node/blockstorage.h \
node/caches.h \
node/chainstate.h \ node/chainstate.h \
node/coin.h \ node/coin.h \
node/coinstats.h \ node/coinstats.h \
@@ -340,6 +341,7 @@ libbitcoin_server_a_SOURCES = \
net.cpp \ net.cpp \
net_processing.cpp \ net_processing.cpp \
node/blockstorage.cpp \ node/blockstorage.cpp \
node/caches.cpp \
node/chainstate.cpp \ node/chainstate.cpp \
node/coin.cpp \ node/coin.cpp \
node/coinstats.cpp \ node/coinstats.cpp \

View File

@@ -34,6 +34,7 @@
#include <net_processing.h> #include <net_processing.h>
#include <netbase.h> #include <netbase.h>
#include <node/blockstorage.h> #include <node/blockstorage.h>
#include <node/caches.h> // for CalculateCacheSizes
#include <node/chainstate.h> // for LoadChainstate #include <node/chainstate.h> // for LoadChainstate
#include <node/context.h> #include <node/context.h>
#include <node/miner.h> #include <node/miner.h>
@@ -1381,36 +1382,20 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false);
// cache size calculations // cache size calculations
int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20); CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size());
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
nTotalCache -= nBlockTreeDBCache;
int64_t nTxIndexCache = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
nTotalCache -= nTxIndexCache;
int64_t filter_index_cache = 0;
if (!g_enabled_filter_types.empty()) {
size_t n_indexes = g_enabled_filter_types.size();
int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
filter_index_cache = max_cache / n_indexes;
nTotalCache -= filter_index_cache * n_indexes;
}
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
nTotalCache -= nCoinDBCache;
int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
int64_t nMempoolSizeMax = args.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t nMempoolSizeMax = args.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
LogPrintf("Cache configuration:\n"); LogPrintf("Cache configuration:\n");
LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for block index database\n", cache_sizes.block_tree_db * (1.0 / 1024 / 1024));
if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for transaction index database\n", cache_sizes.tx_index * (1.0 / 1024 / 1024));
} }
for (BlockFilterType filter_type : g_enabled_filter_types) { for (BlockFilterType filter_type : g_enabled_filter_types) {
LogPrintf("* Using %.1f MiB for %s block filter index database\n", LogPrintf("* Using %.1f MiB for %s block filter index database\n",
filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type)); cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
} }
LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for chain state database\n", cache_sizes.coins_db * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
bool fLoaded = false; bool fLoaded = false;
while (!fLoaded && !ShutdownRequested()) { while (!fLoaded && !ShutdownRequested()) {
@@ -1427,9 +1412,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
fPruneMode, fPruneMode,
chainparams.GetConsensus(), chainparams.GetConsensus(),
fReindexChainState, fReindexChainState,
nBlockTreeDBCache, cache_sizes.block_tree_db,
nCoinDBCache, cache_sizes.coins_db,
nCoinCacheUsage, cache_sizes.coins,
ShutdownRequested, ShutdownRequested,
[]() { []() {
uiInterface.ThreadSafeMessageBox( uiInterface.ThreadSafeMessageBox(
@@ -1548,14 +1533,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
return InitError(*error); return InitError(*error);
} }
g_txindex = std::make_unique<TxIndex>(nTxIndexCache, false, fReindex); g_txindex = std::make_unique<TxIndex>(cache_sizes.tx_index, false, fReindex);
if (!g_txindex->Start(chainman.ActiveChainstate())) { if (!g_txindex->Start(chainman.ActiveChainstate())) {
return false; return false;
} }
} }
for (const auto& filter_type : g_enabled_filter_types) { for (const auto& filter_type : g_enabled_filter_types) {
InitBlockFilterIndex(filter_type, filter_index_cache, false, fReindex); InitBlockFilterIndex(filter_type, cache_sizes.filter_index, false, fReindex);
if (!GetBlockFilterIndex(filter_type)->Start(chainman.ActiveChainstate())) { if (!GetBlockFilterIndex(filter_type)->Start(chainman.ActiveChainstate())) {
return false; return false;
} }

38
src/node/caches.cpp Normal file
View File

@@ -0,0 +1,38 @@
// Copyright (c) 2021 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <node/caches.h>
#include <txdb.h>
#include <util/system.h>
#include <validation.h>
CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
{
int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20);
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
nTotalCache -= nBlockTreeDBCache;
int64_t nTxIndexCache = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
nTotalCache -= nTxIndexCache;
int64_t filter_index_cache = 0;
if (n_indexes > 0) {
int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
filter_index_cache = max_cache / n_indexes;
nTotalCache -= filter_index_cache * n_indexes;
}
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
nTotalCache -= nCoinDBCache;
int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
return {
nBlockTreeDBCache,
nCoinDBCache,
nCoinCacheUsage,
nTxIndexCache,
filter_index_cache,
};
}

22
src/node/caches.h Normal file
View File

@@ -0,0 +1,22 @@
// Copyright (c) 2021 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_NODE_CACHES_H
#define BITCOIN_NODE_CACHES_H
#include <cstddef> // for size_t
#include <cstdint> // for int64_t
class ArgsManager;
struct CacheSizes {
int64_t block_tree_db;
int64_t coins_db;
int64_t coins;
int64_t tx_index;
int64_t filter_index;
};
CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
#endif // BITCOIN_NODE_CACHES_H