highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-03-27 07:55:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge bitcoin/bitcoin#19521: Coinstats Index

Browse Source 
5f96d7d22d rpc: gettxoutsetinfo rejects hash_serialized_2 for specific height (Fabian Jahr)
23fe50436b test: Add test for coinstatsindex behavior in reorgs (Fabian Jahr)
90c966b0f3 rpc: Allow gettxoutsetinfo and getblockstats for stale blocks (Fabian Jahr)
b9362392ae index, rpc: Add use_index option for gettxoutsetinfo (Fabian Jahr)
bb7788b121 test: Test coinstatsindex robustness across restarts (Fabian Jahr)
e0938c2909 test: Add tests for block_info in gettxoutsetinfo (Fabian Jahr)
2501576ecc rpc, index: Add verbose amounts tracking to Coinstats index (Fabian Jahr)
655d929836 test: add coinstatsindex getindexinfo coverage, improve current tests (Jon Atack)
ca01bb8d68 rpc: Add Coinstats index to getindexinfo (Fabian Jahr)
57a026c30f test: Add unit test for Coinstats index (Fabian Jahr)
6a4c0c09ab test: Add functional test for Coinstats index (Fabian Jahr)
3f166ecc12 rpc: gettxoutsetinfo can be requested for specific blockheights (Fabian Jahr)
3c914d58ff index: Coinstats index can be activated with command line flag (Fabian Jahr)
dd58a4de21 index: Add Coinstats index (Fabian Jahr)
a8a46c4b3c refactor: Simplify ApplyStats and ApplyHash (Fabian Jahr)
9c8a265fd2 refactor: Pass hash_type to CoinsStats in stats object (Fabian Jahr)
2e2648a902 crypto: Make MuHash Remove method efficient (Fabian Jahr)

Pull request description:

  This is part of the coinstats index project tracked in #18000

  While the review of the new UTXO set hash algorithm (MuHash) takes longer recently #19328 was merged which added the possibility to run `gettxoutsetinfo` with a specific hash type. As the first type it added `hash_type=none` which skips the hashing of the UTXO set altogether. This alone did not make `gettxoutsetinfo` much faster but it allows the use of an index for the remaining coin statistics even before a new hashing algorithm has been added. Credit to Sjors for the idea to take this intermediate step.

  Features summary:
  - Users can start their node with the option `-coinstatsindex` which syncs the index in the background
  - After the index is synced the user can  use `gettxoutsetinfo` with `hash_type=none` or `hash_type=muhash` and will get the response instantly out of the index
  - The user can specify a height or block hash when calling `gettxoutsetinfo` to see coin statistics at a specific block height

ACKs for top commit:
  Sjors:
    re-tACK 5f96d7d22d
  jonatack:
    Code review re-ACK 5f96d7d22d per `git range-diff 13d27b4 07201d3 5f96d7d`
  promag:
    Tested ACK 5f96d7d22d. Light code review ACK 5f96d7d22d.

Tree-SHA512: cbca78bee8e9605c19da4fbcd184625fb280200718396c694a56c7daab6f44ad23ca9fb5456d09f245d8b8d9659fdc2b3f3ce5e953c1c6cf4003dbc74c0463c2
This commit is contained in:
W. J. van der Laan
2021-04-30 08:31:41 +02:00
parent 480bf01c29 5f96d7d22d
commit 2b45cf0bcd
22 changed files with 1177 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
src/rpc/blockchain.cpp
View File

@@ -14,6 +14,7 @@
#include <core_io.h>
#include <hash.h>
#include <index/blockfilterindex.h>
#include <index/coinstatsindex.h>
#include <node/blockstorage.h>
#include <node/coinstats.h>
#include <node/context.h>
@@ -139,6 +140,33 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b
return blockindex == tip ? 1 : -1;
}
CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) {
LOCK(::cs_main);
CChain& active_chain = chainman.ActiveChain();
if (param.isNum()) {
const int height{param.get_int()};
if (height < 0) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height));
}
const int current_tip{active_chain.Height()};
if (height > current_tip) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip));
}
return active_chain[height];
} else {
const uint256 hash{ParseHashV(param, "hash_or_height")};
CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash);
if (!pindex) {
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
}
return pindex;
}
}
UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex)
{
// Serialize passed information without accessing chain state of the active chain!
@@ -1068,50 +1096,88 @@ static RPCHelpMan gettxoutsetinfo()
{
return RPCHelpMan{"gettxoutsetinfo",
"\nReturns statistics about the unspent transaction output set.\n"
"Note this call may take some time.\n",
"Note this call may take some time if you are not using coinstatsindex.\n",
{
{"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."},
{"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The block hash or height of the target height (only available with coinstatsindex)", "", {"", "string or numeric"}},
{"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."},
},
RPCResult{
RPCResult::Type::OBJ, "", "",
{
{RPCResult::Type::NUM, "height", "The block height (index) of the returned statistics"},
{RPCResult::Type::STR_HEX, "bestblock", "The hash of the block at which these statistics are calculated"},
{RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"},
{RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"},
{RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"},
{RPCResult::Type::NUM, "bogosize", "Database-independent, meaningless metric indicating the UTXO set size"},
{RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"},
{RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"},
{RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"},
{RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"},
{RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"},
{RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"},
{RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"},
{RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)",
{
{RPCResult::Type::STR_AMOUNT, "prevout_spent", ""},
{RPCResult::Type::STR_AMOUNT, "coinbase", ""},
{RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""},
{RPCResult::Type::STR_AMOUNT, "unspendable", ""},
{RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories",
{
{RPCResult::Type::STR_AMOUNT, "genesis_block", ""},
{RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"},
{RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"},
{RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"},
}}
}},
}},
RPCExamples{
HelpExampleCli("gettxoutsetinfo", "")
+ HelpExampleRpc("gettxoutsetinfo", "")
HelpExampleCli("gettxoutsetinfo", "") +
HelpExampleCli("gettxoutsetinfo", R"("none")") +
HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") +
HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") +
HelpExampleRpc("gettxoutsetinfo", "") +
HelpExampleRpc("gettxoutsetinfo", R"("none")") +
HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") +
HelpExampleRpc("gettxoutsetinfo", R"("none", "00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09")")
},
[&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
{
UniValue ret(UniValue::VOBJ);
CCoinsStats stats;
CBlockIndex* pindex{nullptr};
const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())};
CCoinsStats stats{hash_type};
stats.index_requested = request.params[2].isNull() || request.params[2].get_bool();
NodeContext& node = EnsureAnyNodeContext(request.context);
ChainstateManager& chainman = EnsureChainman(node);
CChainState& active_chainstate = chainman.ActiveChainstate();
active_chainstate.ForceFlushStateToDisk();
const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())};
CCoinsView* coins_view;
BlockManager* blockman;
{
LOCK(::cs_main);
coins_view = &active_chainstate.CoinsDB();
blockman = &active_chainstate.m_blockman;
pindex = blockman->LookupBlockIndex(coins_view->GetBestBlock());
}
if (GetUTXOStats(coins_view, *blockman, stats, hash_type, node.rpc_interruption_point)) {
if (!request.params[1].isNull()) {
if (!g_coin_stats_index) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "Querying specific block heights requires coinstatsindex");
}
if (stats.m_hash_type == CoinStatsHashType::HASH_SERIALIZED) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block");
}
pindex = ParseHashOrHeight(request.params[1], chainman);
}
if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point, pindex)) {
ret.pushKV("height", (int64_t)stats.nHeight);
ret.pushKV("bestblock", stats.hashBlock.GetHex());
ret.pushKV("transactions", (int64_t)stats.nTransactions);
ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs);
ret.pushKV("bogosize", (int64_t)stats.nBogoSize);
if (hash_type == CoinStatsHashType::HASH_SERIALIZED) {
@@ -1120,9 +1186,42 @@ static RPCHelpMan gettxoutsetinfo()
if (hash_type == CoinStatsHashType::MUHASH) {
ret.pushKV("muhash", stats.hashSerialized.GetHex());
}
ret.pushKV("disk_size", stats.nDiskSize);
ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount));
if (!stats.index_used) {
ret.pushKV("transactions", static_cast<int64_t>(stats.nTransactions));
ret.pushKV("disk_size", stats.nDiskSize);
} else {
ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount));
CCoinsStats prev_stats{hash_type};
if (pindex->nHeight > 0) {
GetUTXOStats(coins_view, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), prev_stats, node.rpc_interruption_point, pindex->pprev);
}
UniValue block_info(UniValue::VOBJ);
block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount));
block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount));
block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount));
block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount));
UniValue unspendables(UniValue::VOBJ);
unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block));
unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30));
unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts));
unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards));
block_info.pushKV("unspendables", unspendables);
ret.pushKV("block_info", block_info);
}
} else {
if (g_coin_stats_index) {
const IndexSummary summary{g_coin_stats_index->GetSummary()};
if (!summary.synced) {
throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to read UTXO set because coinstatsindex is still syncing. Current height: %d", summary.best_block_height));
}
}
throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
}
return ret;
@@ -1910,31 +2009,7 @@ static RPCHelpMan getblockstats()
{
ChainstateManager& chainman = EnsureAnyChainman(request.context);
LOCK(cs_main);
CChain& active_chain = chainman.ActiveChain();
CBlockIndex* pindex;
if (request.params[0].isNum()) {
const int height = request.params[0].get_int();
const int current_tip = active_chain.Height();
if (height < 0) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height));
}
if (height > current_tip) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip));
}
pindex = active_chain[height];
} else {
const uint256 hash(ParseHashV(request.params[0], "hash_or_height"));
pindex = chainman.m_blockman.LookupBlockIndex(hash);
if (!pindex) {
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
}
if (!active_chain.Contains(pindex)) {
throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString()));
}
}
CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)};
CHECK_NONFATAL(pindex != nullptr);
std::set<std::string> stats;
@@ -2492,7 +2567,7 @@ static RPCHelpMan dumptxoutset()
UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile)
{
std::unique_ptr<CCoinsViewCursor> pcursor;
CCoinsStats stats;
CCoinsStats stats{CoinStatsHashType::NONE};
CBlockIndex* tip;
{
@@ -2512,7 +2587,7 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil
chainstate.ForceFlushStateToDisk();
if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) {
if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, node.rpc_interruption_point)) {
throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
}