highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-11-12 06:58:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge bitcoin/bitcoin#19521: Coinstats Index

Browse Source 
5f96d7d22d rpc: gettxoutsetinfo rejects hash_serialized_2 for specific height (Fabian Jahr)
23fe50436b test: Add test for coinstatsindex behavior in reorgs (Fabian Jahr)
90c966b0f3 rpc: Allow gettxoutsetinfo and getblockstats for stale blocks (Fabian Jahr)
b9362392ae index, rpc: Add use_index option for gettxoutsetinfo (Fabian Jahr)
bb7788b121 test: Test coinstatsindex robustness across restarts (Fabian Jahr)
e0938c2909 test: Add tests for block_info in gettxoutsetinfo (Fabian Jahr)
2501576ecc rpc, index: Add verbose amounts tracking to Coinstats index (Fabian Jahr)
655d929836 test: add coinstatsindex getindexinfo coverage, improve current tests (Jon Atack)
ca01bb8d68 rpc: Add Coinstats index to getindexinfo (Fabian Jahr)
57a026c30f test: Add unit test for Coinstats index (Fabian Jahr)
6a4c0c09ab test: Add functional test for Coinstats index (Fabian Jahr)
3f166ecc12 rpc: gettxoutsetinfo can be requested for specific blockheights (Fabian Jahr)
3c914d58ff index: Coinstats index can be activated with command line flag (Fabian Jahr)
dd58a4de21 index: Add Coinstats index (Fabian Jahr)
a8a46c4b3c refactor: Simplify ApplyStats and ApplyHash (Fabian Jahr)
9c8a265fd2 refactor: Pass hash_type to CoinsStats in stats object (Fabian Jahr)
2e2648a902 crypto: Make MuHash Remove method efficient (Fabian Jahr)

Pull request description:

  This is part of the coinstats index project tracked in #18000

  While the review of the new UTXO set hash algorithm (MuHash) takes longer recently #19328 was merged which added the possibility to run `gettxoutsetinfo` with a specific hash type. As the first type it added `hash_type=none` which skips the hashing of the UTXO set altogether. This alone did not make `gettxoutsetinfo` much faster but it allows the use of an index for the remaining coin statistics even before a new hashing algorithm has been added. Credit to Sjors for the idea to take this intermediate step.

  Features summary:
  - Users can start their node with the option `-coinstatsindex` which syncs the index in the background
  - After the index is synced the user can  use `gettxoutsetinfo` with `hash_type=none` or `hash_type=muhash` and will get the response instantly out of the index
  - The user can specify a height or block hash when calling `gettxoutsetinfo` to see coin statistics at a specific block height

ACKs for top commit:
  Sjors:
    re-tACK 5f96d7d22d
  jonatack:
    Code review re-ACK 5f96d7d22d per `git range-diff 13d27b4 07201d3 5f96d7d`
  promag:
    Tested ACK 5f96d7d22d. Light code review ACK 5f96d7d22d.

Tree-SHA512: cbca78bee8e9605c19da4fbcd184625fb280200718396c694a56c7daab6f44ad23ca9fb5456d09f245d8b8d9659fdc2b3f3ce5e953c1c6cf4003dbc74c0463c2
This commit is contained in:
W. J. van der Laan
2021-04-30 08:31:41 +02:00
parent 480bf01c29 5f96d7d22d
commit 2b45cf0bcd
22 changed files with 1177 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
src/node/coinstats.cpp
View File

@@ -8,6 +8,7 @@
#include <coins.h>
#include <crypto/muhash.h>
#include <hash.h>
#include <index/coinstatsindex.h>
#include <serialize.h>
#include <uint256.h>
#include <util/system.h>
@@ -16,44 +17,22 @@
#include <map>
// Database-independent metric indicating the UTXO set size
static uint64_t GetBogoSize(const CScript& scriptPubKey)
uint64_t GetBogoSize(const CScript& script_pub_key)
{
return 32 /* txid */ +
4 /* vout index */ +
4 /* height + coinbase */ +
8 /* amount */ +
2 /* scriptPubKey len */ +
scriptPubKey.size() /* scriptPubKey */;
script_pub_key.size() /* scriptPubKey */;
}
static void ApplyHash(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it)
{
if (it == outputs.begin()) {
ss << hash;
ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u);
}
ss << VARINT(it->first + 1);
ss << it->second.out.scriptPubKey;
ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED);
if (it == std::prev(outputs.end())) {
ss << VARINT(0u);
}
}
static void ApplyHash(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) {}
static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it)
{
COutPoint outpoint = COutPoint(hash, it->first);
Coin coin = it->second;
CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) {
CDataStream ss(SER_DISK, PROTOCOL_VERSION);
ss << outpoint;
ss << static_cast<uint32_t>(coin.nHeight * 2 + coin.fCoinBase);
ss << coin.out;
muhash.Insert(MakeUCharSpan(ss));
return ss;
}
//! Warning: be very careful when changing this! assumeutxo and UTXO snapshot
@@ -68,14 +47,40 @@ static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& has
//! It is also possible, though very unlikely, that a change in this
//! construction could cause a previously invalid (and potentially malicious)
//! UTXO snapshot to be considered valid.
template <typename T>
static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs)
static void ApplyHash(CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs)
{
for (auto it = outputs.begin(); it != outputs.end(); ++it) {
if (it == outputs.begin()) {
ss << hash;
ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u);
}
ss << VARINT(it->first + 1);
ss << it->second.out.scriptPubKey;
ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED);
if (it == std::prev(outputs.end())) {
ss << VARINT(0u);
}
}
}
static void ApplyHash(std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) {}
static void ApplyHash(MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs)
{
for (auto it = outputs.begin(); it != outputs.end(); ++it) {
COutPoint outpoint = COutPoint(hash, it->first);
Coin coin = it->second;
muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin)));
}
}
static void ApplyStats(CCoinsStats& stats, const uint256& hash, const std::map<uint32_t, Coin>& outputs)
{
assert(!outputs.empty());
stats.nTransactions++;
for (auto it = outputs.begin(); it != outputs.end(); ++it) {
ApplyHash(stats, hash_obj, hash, outputs, it);
stats.nTransactionOutputs++;
stats.nTotalAmount += it->second.out.nValue;
stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey);
@@ -84,18 +89,25 @@ static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, con
//! Calculate statistics about the unspent transaction output set
template <typename T>
static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point)
static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point, const CBlockIndex* pindex)
{
stats = CCoinsStats();
std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
assert(pcursor);
stats.hashBlock = pcursor->GetBestBlock();
{
LOCK(cs_main);
assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman));
const CBlockIndex* block = blockman.LookupBlockIndex(stats.hashBlock);
stats.nHeight = Assert(block)->nHeight;
if (!pindex) {
{
LOCK(cs_main);
assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman));
pindex = blockman.LookupBlockIndex(view->GetBestBlock());
}
}
stats.nHeight = Assert(pindex)->nHeight;
stats.hashBlock = pindex->GetBlockHash();
// Use CoinStatsIndex if it is requested and available and a hash_type of Muhash or None was requested
if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index && stats.index_requested) {
stats.index_used = true;
return g_coin_stats_index->LookUpStats(pindex, stats);
}
PrepareHash(hash_obj, stats);
@@ -108,7 +120,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats&
Coin coin;
if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
if (!outputs.empty() && key.hash != prevkey) {
ApplyStats(stats, hash_obj, prevkey, outputs);
ApplyStats(stats, prevkey, outputs);
ApplyHash(hash_obj, prevkey, outputs);
outputs.clear();
}
prevkey = key.hash;
@@ -120,7 +133,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats&
pcursor->Next();
}
if (!outputs.empty()) {
ApplyStats(stats, hash_obj, prevkey, outputs);
ApplyStats(stats, prevkey, outputs);
ApplyHash(hash_obj, prevkey, outputs);
}
FinalizeHash(hash_obj, stats);
@@ -129,19 +143,19 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats&
return true;
}
bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point)
bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point, const CBlockIndex* pindex)
{
switch (hash_type) {
switch (stats.m_hash_type) {
case(CoinStatsHashType::HASH_SERIALIZED): {
CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION);
return GetUTXOStats(view, blockman, stats, ss, interruption_point);
return GetUTXOStats(view, blockman, stats, ss, interruption_point, pindex);
}
case(CoinStatsHashType::MUHASH): {
MuHash3072 muhash;
return GetUTXOStats(view, blockman, stats, muhash, interruption_point);
return GetUTXOStats(view, blockman, stats, muhash, interruption_point, pindex);
}
case(CoinStatsHashType::NONE): {
return GetUTXOStats(view, blockman, stats, nullptr, interruption_point);
return GetUTXOStats(view, blockman, stats, nullptr, interruption_point, pindex);
}
} // no default case, so the compiler can warn about missing cases
assert(false);

28
src/node/coinstats.h
View File

@@ -7,6 +7,9 @@
#define BITCOIN_NODE_COINSTATS_H
#include <amount.h>
#include <chain.h>
#include <coins.h>
#include <streams.h>
#include <uint256.h>
#include <cstdint>
@@ -23,6 +26,7 @@ enum class CoinStatsHashType {
struct CCoinsStats
{
CoinStatsHashType m_hash_type;
int nHeight{0};
uint256 hashBlock{};
uint64_t nTransactions{0};
@@ -34,9 +38,31 @@ struct CCoinsStats
//! The number of coins contained.
uint64_t coins_count{0};
//! Signals if the coinstatsindex should be used (when available).
bool index_requested{true};
//! Signals if the coinstatsindex was used to retrieve the statistics.
bool index_used{false};
// Following values are only available from coinstats index
CAmount total_subsidy{0};
CAmount block_unspendable_amount{0};
CAmount block_prevout_spent_amount{0};
CAmount block_new_outputs_ex_coinbase_amount{0};
CAmount block_coinbase_amount{0};
CAmount unspendables_genesis_block{0};
CAmount unspendables_bip30{0};
CAmount unspendables_scripts{0};
CAmount unspendables_unclaimed_rewards{0};
CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {}
};
//! Calculate statistics about the unspent transaction output set
bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function<void()>& interruption_point = {});
bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point = {}, const CBlockIndex* pindex = nullptr);
uint64_t GetBogoSize(const CScript& script_pub_key);
CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin);
#endif // BITCOIN_NODE_COINSTATS_H