From 8ea8c927ac05980d6a81252e40b7444e9abb74f9 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 24 May 2021 00:46:08 +0200 Subject: [PATCH 1/8] index: Avoid unnecessary type casts in coinstatsindex --- src/index/coinstatsindex.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index e046527283a..5e3f7602c85 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -143,10 +143,10 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) continue; } - for (size_t j = 0; j < tx->vout.size(); ++j) { + for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut& out{tx->vout[j]}; Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; - COutPoint outpoint{tx->GetHash(), static_cast(j)}; + COutPoint outpoint{tx->GetHash(), j}; // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { @@ -402,9 +402,9 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex for (size_t i = 0; i < block.vtx.size(); ++i) { const auto& tx{block.vtx.at(i)}; - for (size_t j = 0; j < tx->vout.size(); ++j) { + for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut& out{tx->vout[j]}; - COutPoint outpoint{tx->GetHash(), static_cast(j)}; + COutPoint outpoint{tx->GetHash(), j}; Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; // Skip unspendable coins From fb65dde147f63422c4148b089c2f5be0bf5ba80f Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Thu, 3 Jun 2021 01:31:47 +0200 Subject: [PATCH 2/8] scripted-diff: Fix coinstats data member names Initially these values were 'per block' in an earlier version but were then changed to total values. The names were not updated to reflect that. -BEGIN VERIFY SCRIPT- s() { git grep -l "$1" src | xargs sed -i "s/$1/$2/g"; } s 'm_block_unspendable_amount' 'm_total_unspendable_amount' s 'm_block_prevout_spent_amount' 'm_total_prevout_spent_amount' s 'm_block_new_outputs_ex_coinbase_amount' 'm_total_new_outputs_ex_coinbase_amount' s 'm_block_coinbase_amount' 'm_total_coinbase_amount' s 'block_unspendable_amount' 'total_unspendable_amount' s 'block_prevout_spent_amount' 'total_prevout_spent_amount' s 'block_new_outputs_ex_coinbase_amount' 'total_new_outputs_ex_coinbase_amount' s 'block_coinbase_amount' 'total_coinbase_amount' s 'unspendables_genesis_block' 'total_unspendables_genesis_block' s 'unspendables_bip30' 'total_unspendables_bip30' s 'unspendables_scripts' 'total_unspendables_scripts' s 'unspendables_unclaimed_rewards' 'total_unspendables_unclaimed_rewards' s 'm_unspendables_genesis_block' 'm_total_unspendables_genesis_block' s 'm_unspendables_bip30' 'm_total_unspendables_bip30' s 'm_unspendables_scripts' 'm_total_unspendables_scripts' s 'm_unspendables_unclaimed_rewards' 'm_total_unspendables_unclaimed_rewards' -END VERIFY SCRIPT- --- src/index/coinstatsindex.cpp | 136 +++++++++++++++++------------------ src/index/coinstatsindex.h | 16 ++--- src/node/coinstats.h | 16 ++--- src/rpc/blockchain.cpp | 18 ++--- 4 files changed, 93 insertions(+), 93 deletions(-) diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 5e3f7602c85..cb940234e20 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -24,14 +24,14 @@ struct DBVal { uint64_t bogo_size; CAmount total_amount; CAmount total_subsidy; - CAmount block_unspendable_amount; - CAmount block_prevout_spent_amount; - CAmount block_new_outputs_ex_coinbase_amount; - CAmount block_coinbase_amount; - CAmount unspendables_genesis_block; - CAmount unspendables_bip30; - CAmount unspendables_scripts; - CAmount unspendables_unclaimed_rewards; + CAmount total_unspendable_amount; + CAmount total_prevout_spent_amount; + CAmount total_new_outputs_ex_coinbase_amount; + CAmount total_coinbase_amount; + CAmount total_unspendables_genesis_block; + CAmount total_unspendables_bip30; + CAmount total_unspendables_scripts; + CAmount total_unspendables_unclaimed_rewards; SERIALIZE_METHODS(DBVal, obj) { @@ -40,14 +40,14 @@ struct DBVal { READWRITE(obj.bogo_size); READWRITE(obj.total_amount); READWRITE(obj.total_subsidy); - READWRITE(obj.block_unspendable_amount); - READWRITE(obj.block_prevout_spent_amount); - READWRITE(obj.block_new_outputs_ex_coinbase_amount); - READWRITE(obj.block_coinbase_amount); - READWRITE(obj.unspendables_genesis_block); - READWRITE(obj.unspendables_bip30); - READWRITE(obj.unspendables_scripts); - READWRITE(obj.unspendables_unclaimed_rewards); + READWRITE(obj.total_unspendable_amount); + READWRITE(obj.total_prevout_spent_amount); + READWRITE(obj.total_new_outputs_ex_coinbase_amount); + READWRITE(obj.total_coinbase_amount); + READWRITE(obj.total_unspendables_genesis_block); + READWRITE(obj.total_unspendables_bip30); + READWRITE(obj.total_unspendables_scripts); + READWRITE(obj.total_unspendables_unclaimed_rewards); } }; @@ -138,8 +138,8 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { - m_block_unspendable_amount += block_subsidy; - m_unspendables_bip30 += block_subsidy; + m_total_unspendable_amount += block_subsidy; + m_total_unspendables_bip30 += block_subsidy; continue; } @@ -150,17 +150,17 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { - m_block_unspendable_amount += coin.out.nValue; - m_unspendables_scripts += coin.out.nValue; + m_total_unspendable_amount += coin.out.nValue; + m_total_unspendables_scripts += coin.out.nValue; continue; } m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); if (tx->IsCoinBase()) { - m_block_coinbase_amount += coin.out.nValue; + m_total_coinbase_amount += coin.out.nValue; } else { - m_block_new_outputs_ex_coinbase_amount += coin.out.nValue; + m_total_new_outputs_ex_coinbase_amount += coin.out.nValue; } ++m_transaction_output_count; @@ -178,7 +178,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); - m_block_prevout_spent_amount += coin.out.nValue; + m_total_prevout_spent_amount += coin.out.nValue; --m_transaction_output_count; m_total_amount -= coin.out.nValue; @@ -188,17 +188,17 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) } } else { // genesis block - m_block_unspendable_amount += block_subsidy; - m_unspendables_genesis_block += block_subsidy; + m_total_unspendable_amount += block_subsidy; + m_total_unspendables_genesis_block += block_subsidy; } // If spent prevouts + block subsidy are still a higher amount than // new outputs + coinbase + current unspendable amount this means // the miner did not claim the full block reward. Unclaimed block // rewards are also unspendable. - const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)}; - m_block_unspendable_amount += unclaimed_rewards; - m_unspendables_unclaimed_rewards += unclaimed_rewards; + const CAmount unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)}; + m_total_unspendable_amount += unclaimed_rewards; + m_total_unspendables_unclaimed_rewards += unclaimed_rewards; std::pair value; value.first = pindex->GetBlockHash(); @@ -206,14 +206,14 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; value.second.total_subsidy = m_total_subsidy; - value.second.block_unspendable_amount = m_block_unspendable_amount; - value.second.block_prevout_spent_amount = m_block_prevout_spent_amount; - value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount; - value.second.block_coinbase_amount = m_block_coinbase_amount; - value.second.unspendables_genesis_block = m_unspendables_genesis_block; - value.second.unspendables_bip30 = m_unspendables_bip30; - value.second.unspendables_scripts = m_unspendables_scripts; - value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards; + value.second.total_unspendable_amount = m_total_unspendable_amount; + value.second.total_prevout_spent_amount = m_total_prevout_spent_amount; + value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount; + value.second.total_coinbase_amount = m_total_coinbase_amount; + value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block; + value.second.total_unspendables_bip30 = m_total_unspendables_bip30; + value.second.total_unspendables_scripts = m_total_unspendables_scripts; + value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards; uint256 out; m_muhash.Finalize(out); @@ -317,14 +317,14 @@ bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& co coins_stats.nBogoSize = entry.bogo_size; coins_stats.nTotalAmount = entry.total_amount; coins_stats.total_subsidy = entry.total_subsidy; - coins_stats.block_unspendable_amount = entry.block_unspendable_amount; - coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount; - coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; - coins_stats.block_coinbase_amount = entry.block_coinbase_amount; - coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block; - coins_stats.unspendables_bip30 = entry.unspendables_bip30; - coins_stats.unspendables_scripts = entry.unspendables_scripts; - coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + coins_stats.total_unspendable_amount = entry.total_unspendable_amount; + coins_stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; + coins_stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; + coins_stats.total_coinbase_amount = entry.total_coinbase_amount; + coins_stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block; + coins_stats.total_unspendables_bip30 = entry.total_unspendables_bip30; + coins_stats.total_unspendables_scripts = entry.total_unspendables_scripts; + coins_stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; return true; } @@ -354,14 +354,14 @@ bool CoinStatsIndex::Init() m_bogo_size = entry.bogo_size; m_total_amount = entry.total_amount; m_total_subsidy = entry.total_subsidy; - m_block_unspendable_amount = entry.block_unspendable_amount; - m_block_prevout_spent_amount = entry.block_prevout_spent_amount; - m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; - m_block_coinbase_amount = entry.block_coinbase_amount; - m_unspendables_genesis_block = entry.unspendables_genesis_block; - m_unspendables_bip30 = entry.unspendables_bip30; - m_unspendables_scripts = entry.unspendables_scripts; - m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + m_total_unspendable_amount = entry.total_unspendable_amount; + m_total_prevout_spent_amount = entry.total_prevout_spent_amount; + m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; + m_total_coinbase_amount = entry.total_coinbase_amount; + m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; + m_total_unspendables_bip30 = entry.total_unspendables_bip30; + m_total_unspendables_scripts = entry.total_unspendables_scripts; + m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; } return true; @@ -409,17 +409,17 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { - m_block_unspendable_amount -= coin.out.nValue; - m_unspendables_scripts -= coin.out.nValue; + m_total_unspendable_amount -= coin.out.nValue; + m_total_unspendables_scripts -= coin.out.nValue; continue; } m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); if (tx->IsCoinBase()) { - m_block_coinbase_amount -= coin.out.nValue; + m_total_coinbase_amount -= coin.out.nValue; } else { - m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue; + m_total_new_outputs_ex_coinbase_amount -= coin.out.nValue; } --m_transaction_output_count; @@ -437,7 +437,7 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); - m_block_prevout_spent_amount -= coin.out.nValue; + m_total_prevout_spent_amount -= coin.out.nValue; m_transaction_output_count++; m_total_amount += coin.out.nValue; @@ -446,9 +446,9 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex } } - const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)}; - m_block_unspendable_amount -= unclaimed_rewards; - m_unspendables_unclaimed_rewards -= unclaimed_rewards; + const CAmount unclaimed_rewards{(m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)}; + m_total_unspendable_amount -= unclaimed_rewards; + m_total_unspendables_unclaimed_rewards -= unclaimed_rewards; // Check that the rolled back internal values are consistent with the DB read out uint256 out; @@ -459,14 +459,14 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex Assert(m_total_amount == read_out.second.total_amount); Assert(m_bogo_size == read_out.second.bogo_size); Assert(m_total_subsidy == read_out.second.total_subsidy); - Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount); - Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount); - Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount); - Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount); - Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block); - Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30); - Assert(m_unspendables_scripts == read_out.second.unspendables_scripts); - Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards); + Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount); + Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount); + Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount); + Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount); + Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block); + Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30); + Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts); + Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards); return m_db->Write(DB_MUHASH, m_muhash); } diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 6149f9b4b3d..a575b37c7c4 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -25,14 +25,14 @@ private: uint64_t m_bogo_size{0}; CAmount m_total_amount{0}; CAmount m_total_subsidy{0}; - CAmount m_block_unspendable_amount{0}; - CAmount m_block_prevout_spent_amount{0}; - CAmount m_block_new_outputs_ex_coinbase_amount{0}; - CAmount m_block_coinbase_amount{0}; - CAmount m_unspendables_genesis_block{0}; - CAmount m_unspendables_bip30{0}; - CAmount m_unspendables_scripts{0}; - CAmount m_unspendables_unclaimed_rewards{0}; + CAmount m_total_unspendable_amount{0}; + CAmount m_total_prevout_spent_amount{0}; + CAmount m_total_new_outputs_ex_coinbase_amount{0}; + CAmount m_total_coinbase_amount{0}; + CAmount m_total_unspendables_genesis_block{0}; + CAmount m_total_unspendables_bip30{0}; + CAmount m_total_unspendables_scripts{0}; + CAmount m_total_unspendables_unclaimed_rewards{0}; bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 8be256edc93..ae2e46e4d9b 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -46,14 +46,14 @@ struct CCoinsStats // Following values are only available from coinstats index CAmount total_subsidy{0}; - CAmount block_unspendable_amount{0}; - CAmount block_prevout_spent_amount{0}; - CAmount block_new_outputs_ex_coinbase_amount{0}; - CAmount block_coinbase_amount{0}; - CAmount unspendables_genesis_block{0}; - CAmount unspendables_bip30{0}; - CAmount unspendables_scripts{0}; - CAmount unspendables_unclaimed_rewards{0}; + CAmount total_unspendable_amount{0}; + CAmount total_prevout_spent_amount{0}; + CAmount total_new_outputs_ex_coinbase_amount{0}; + CAmount total_coinbase_amount{0}; + CAmount total_unspendables_genesis_block{0}; + CAmount total_unspendables_bip30{0}; + CAmount total_unspendables_scripts{0}; + CAmount total_unspendables_unclaimed_rewards{0}; CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 03f28239ba4..ee2f5a549bc 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1191,7 +1191,7 @@ static RPCHelpMan gettxoutsetinfo() ret.pushKV("transactions", static_cast(stats.nTransactions)); ret.pushKV("disk_size", stats.nDiskSize); } else { - ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount)); + ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.total_unspendable_amount)); CCoinsStats prev_stats{hash_type}; @@ -1200,16 +1200,16 @@ static RPCHelpMan gettxoutsetinfo() } UniValue block_info(UniValue::VOBJ); - block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount)); - block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount)); - block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount)); - block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount)); + block_info.pushKV("prevout_spent", ValueFromAmount(stats.total_prevout_spent_amount - prev_stats.total_prevout_spent_amount)); + block_info.pushKV("coinbase", ValueFromAmount(stats.total_coinbase_amount - prev_stats.total_coinbase_amount)); + block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.total_new_outputs_ex_coinbase_amount - prev_stats.total_new_outputs_ex_coinbase_amount)); + block_info.pushKV("unspendable", ValueFromAmount(stats.total_unspendable_amount - prev_stats.total_unspendable_amount)); UniValue unspendables(UniValue::VOBJ); - unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block)); - unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30)); - unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts)); - unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards)); + unspendables.pushKV("genesis_block", ValueFromAmount(stats.total_unspendables_genesis_block - prev_stats.total_unspendables_genesis_block)); + unspendables.pushKV("bip30", ValueFromAmount(stats.total_unspendables_bip30 - prev_stats.total_unspendables_bip30)); + unspendables.pushKV("scripts", ValueFromAmount(stats.total_unspendables_scripts - prev_stats.total_unspendables_scripts)); + unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.total_unspendables_unclaimed_rewards - prev_stats.total_unspendables_unclaimed_rewards)); block_info.pushKV("unspendables", unspendables); ret.pushKV("block_info", block_info); From 1e3842385b8c0d15086c7cd8736f8c67e6c0c285 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 24 May 2021 01:18:51 +0200 Subject: [PATCH 3/8] index: Use batch writing in coinstatsindex WriteBlock --- src/index/coinstatsindex.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index cb940234e20..084e6b9925a 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -219,7 +219,10 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_muhash.Finalize(out); value.second.muhash = out; - return m_db->Write(DBHeightKey(pindex->nHeight), value) && m_db->Write(DB_MUHASH, m_muhash); + CDBBatch batch(*m_db); + batch.Write(DBHeightKey(pindex->nHeight), value); + batch.Write(DB_MUHASH, m_muhash); + return m_db->WriteBatch(batch); } static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, From 01386bfd88019397237256cb16f91de346eb66f2 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 24 May 2021 01:24:05 +0200 Subject: [PATCH 4/8] Index: Return early from failed coinstatsindex init --- src/index/coinstatsindex.cpp | 40 +++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 084e6b9925a..b3f5d75fb3d 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -344,33 +344,31 @@ bool CoinStatsIndex::Init() } } - if (BaseIndex::Init()) { - const CBlockIndex* pindex{CurrentIndex()}; + if (!BaseIndex::Init()) return false; - if (pindex) { - DBVal entry; - if (!LookUpOne(*m_db, pindex, entry)) { - return false; - } + const CBlockIndex* pindex{CurrentIndex()}; - m_transaction_output_count = entry.transaction_output_count; - m_bogo_size = entry.bogo_size; - m_total_amount = entry.total_amount; - m_total_subsidy = entry.total_subsidy; - m_total_unspendable_amount = entry.total_unspendable_amount; - m_total_prevout_spent_amount = entry.total_prevout_spent_amount; - m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; - m_total_coinbase_amount = entry.total_coinbase_amount; - m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; - m_total_unspendables_bip30 = entry.total_unspendables_bip30; - m_total_unspendables_scripts = entry.total_unspendables_scripts; - m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; + if (pindex) { + DBVal entry; + if (!LookUpOne(*m_db, pindex, entry)) { + return false; } - return true; + m_transaction_output_count = entry.transaction_output_count; + m_bogo_size = entry.bogo_size; + m_total_amount = entry.total_amount; + m_total_subsidy = entry.total_subsidy; + m_total_unspendable_amount = entry.total_unspendable_amount; + m_total_prevout_spent_amount = entry.total_prevout_spent_amount; + m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; + m_total_coinbase_amount = entry.total_coinbase_amount; + m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; + m_total_unspendables_bip30 = entry.total_unspendables_bip30; + m_total_unspendables_scripts = entry.total_unspendables_scripts; + m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; } - return false; + return true; } // Reverse a single block as part of a reorg From a5f6791139554936d13f367660283899a37ff5c7 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 24 May 2021 01:40:17 +0200 Subject: [PATCH 5/8] rpc: Add missing gettxoutsetinfo help docs --- src/rpc/blockchain.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ee2f5a549bc..86f6a4320e7 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1117,13 +1117,13 @@ static RPCHelpMan gettxoutsetinfo() {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)", { - {RPCResult::Type::STR_AMOUNT, "prevout_spent", ""}, - {RPCResult::Type::STR_AMOUNT, "coinbase", ""}, - {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""}, - {RPCResult::Type::STR_AMOUNT, "unspendable", ""}, + {RPCResult::Type::STR_AMOUNT, "prevout_spent", "Total amount of all prevouts spent in this block"}, + {RPCResult::Type::STR_AMOUNT, "coinbase", "Coinbase subsidy amount of this block"}, + {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", "Total amount of new outputs created by this block"}, + {RPCResult::Type::STR_AMOUNT, "unspendable", "Total amount of unspendable outputs created in this block"}, {RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories", { - {RPCResult::Type::STR_AMOUNT, "genesis_block", ""}, + {RPCResult::Type::STR_AMOUNT, "genesis_block", "The unspendable amount of the Genesis block subsidy"}, {RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"}, {RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"}, {RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"}, From d4356d4e48f59c63894b68691cc21ed4892ee716 Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 24 May 2021 17:04:58 +0200 Subject: [PATCH 6/8] rpc: Block until synced if coinstatsindex is used in gettxoutsetinfo During initial sync after startup the gettxoutsetinfo RPC will still return an error while catching up. However, after the initial sync the index will not error immediately anymore when it's in the process of syncing to the tip while being called. Instead it will block until synced and then return the response. --- src/rpc/blockchain.cpp | 19 ++++++++++++------- test/functional/feature_coinstatsindex.py | 14 +------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 86f6a4320e7..2347fcbb736 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1175,6 +1175,18 @@ static RPCHelpMan gettxoutsetinfo() pindex = ParseHashOrHeight(request.params[1], chainman); } + if (stats.index_requested && g_coin_stats_index) { + if (!g_coin_stats_index->BlockUntilSyncedToCurrentChain()) { + const IndexSummary summary{g_coin_stats_index->GetSummary()}; + + // If a specific block was requested and the index has already synced past that height, we can return the + // data already even though the index is not fully synced yet. + if (pindex->nHeight > summary.best_block_height) { + throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to get data because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); + } + } + } + if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point, pindex)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); @@ -1215,13 +1227,6 @@ static RPCHelpMan gettxoutsetinfo() ret.pushKV("block_info", block_info); } } else { - if (g_coin_stats_index) { - const IndexSummary summary{g_coin_stats_index->GetSummary()}; - - if (!summary.synced) { - throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to read UTXO set because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); - } - } throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } return ret; diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index d3adde5cc5a..c2bc485d6ba 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -32,7 +32,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - try_rpc, ) class CoinStatsIndexTest(BitcoinTestFramework): @@ -76,13 +75,11 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.sync_blocks(timeout=120) self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo)) res0 = node.gettxoutsetinfo('none') # The fields 'disk_size' and 'transactions' do not exist on the index del res0['disk_size'], res0['transactions'] - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: res1 = index_node.gettxoutsetinfo(hash_option) # The fields 'block_info' and 'total_unspendable_amount' only exist on the index @@ -97,7 +94,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate a new tip node.generate(5) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: # Fetch old stats by height res2 = index_node.gettxoutsetinfo(hash_option, 102) @@ -176,7 +172,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: # Check all amounts were registered correctly res6 = index_node.gettxoutsetinfo(hash_option, 108) @@ -209,7 +204,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.nodes[0].submitblock(ToHex(block)) self.sync_all() - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: res7 = index_node.gettxoutsetinfo(hash_option, 109) assert_equal(res7['total_unspendable_amount'], Decimal('80.98999999')) @@ -235,7 +229,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(res8, res9) index_node.generate(1) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) @@ -256,14 +249,12 @@ class CoinStatsIndexTest(BitcoinTestFramework): index_node = self.nodes[1] reorg_blocks = index_node.generatetoaddress(2, index_node.getnewaddress()) reorg_block = reorg_blocks[1] - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) # Add two new blocks block = index_node.generate(2)[1] - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) # Test that the result of the reorged block is not returned for its old block height @@ -284,9 +275,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Ensure that removing and re-adding blocks yields consistent results block = index_node.getblockhash(99) index_node.invalidateblock(block) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) index_node.reconsiderblock(block) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) res3 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) assert_equal(res2, res3) @@ -296,8 +285,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): node.getblock(reorg_block) self.restart_node(0, ["-coinstatsindex"]) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash')) - assert_raises_rpc_error(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash', reorg_block) + assert_raises_rpc_error(-32603, "Unable to get data because coinstatsindex is still syncing.", node.gettxoutsetinfo, 'muhash', reorg_block) def _test_index_rejects_hash_serialized(self): self.log.info("Test that the rpc raises if the legacy hash is passed with the index") From 5b3d4e724f377834e24b1f014787cc7aa7fc30fe Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Mon, 24 May 2021 18:37:40 +0200 Subject: [PATCH 7/8] Index: Improve logging in coinstatsindex More accurate logging of a warning should make clear if the recovery condition was hit while catching the results of the previous block. --- src/index/coinstatsindex.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index b3f5d75fb3d..849ea752f27 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -122,9 +122,12 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { + LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n", + read_out.first.ToString(), expected_block_hash.ToString()); + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { - return error("%s: previous block header belongs to unexpected block %s; expected %s", - __func__, read_out.first.ToString(), expected_block_hash.ToString()); + return error("%s: previous block header not found; expected %s", + __func__, expected_block_hash.ToString()); } } @@ -392,9 +395,12 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { + LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n", + read_out.first.ToString(), expected_block_hash.ToString()); + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { - return error("%s: previous block header belongs to unexpected block %s; expected %s", - __func__, read_out.first.ToString(), expected_block_hash.ToString()); + return error("%s: previous block header not found; expected %s", + __func__, expected_block_hash.ToString()); } } } From 779e638ca9b2b37c247577d225b93ac762b0602f Mon Sep 17 00:00:00 2001 From: Fabian Jahr Date: Thu, 3 Jun 2021 02:06:00 +0200 Subject: [PATCH 8/8] coinstats: Add comments for new coinstatsindex values --- src/node/coinstats.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/node/coinstats.h b/src/node/coinstats.h index ae2e46e4d9b..69e856dd15a 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -45,14 +45,24 @@ struct CCoinsStats bool index_used{false}; // Following values are only available from coinstats index + + //! Total cumulative amount of block subsidies up to and including this block CAmount total_subsidy{0}; + //! Total cumulative amount of unspendable coins up to and including this block CAmount total_unspendable_amount{0}; + //! Total cumulative amount of prevouts spent up to and including this block CAmount total_prevout_spent_amount{0}; + //! Total cumulative amount of outputs created up to and including this block CAmount total_new_outputs_ex_coinbase_amount{0}; + //! Total cumulative amount of coinbase outputs up to and including this block CAmount total_coinbase_amount{0}; + //! The unspendable coinbase amount from the genesis block CAmount total_unspendables_genesis_block{0}; + //! The two unspendable coinbase outputs total amount caused by BIP30 CAmount total_unspendables_bip30{0}; + //! Total cumulative amount of outputs sent to unspendable scripts (OP_RETURN for example) up to and including this block CAmount total_unspendables_scripts{0}; + //! Total cumulative amount of coins lost due to unclaimed miner rewards up to and including this block CAmount total_unspendables_unclaimed_rewards{0}; CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {}