diff --git a/src/policy/fees/estimator_man.cpp b/src/policy/fees/estimator_man.cpp index 9b5c5093c46..4c67317d0dc 100644 --- a/src/policy/fees/estimator_man.cpp +++ b/src/policy/fees/estimator_man.cpp @@ -72,9 +72,10 @@ void FeeRateEstimatorManager::TransactionRemovedFromMempool(const CTransactionRe m_block_policy_estimator->removeTx(tx->GetHash()); } -void FeeRateEstimatorManager::MempoolTransactionsRemovedForBlock(const std::shared_ptr& /*block*/, const std::vector& txs_removed_for_block, unsigned int block_height) +void FeeRateEstimatorManager::MempoolTransactionsRemovedForBlock(const std::shared_ptr& block, const std::vector& txs_removed_for_block, unsigned int block_height) { m_block_policy_estimator->processBlock(txs_removed_for_block, block_height); + m_mempool_estimator->MempoolTxsRemovedForBlock(block, txs_removed_for_block, block_height); } CFeeRate FeeRateEstimatorManager::BlockPolicyEstimateRawFee(unsigned int target, double threshold, FeeEstimateHorizon horizon, EstimationResult* buckets) const diff --git a/src/policy/fees/mempool_estimator.cpp b/src/policy/fees/mempool_estimator.cpp index 7c9af99cb7a..ca8912248e4 100644 --- a/src/policy/fees/mempool_estimator.cpp +++ b/src/policy/fees/mempool_estimator.cpp @@ -17,8 +17,47 @@ #include #include +#include +#include +#include +#include +#include #include +namespace { +void AddMinedBlockStats(std::vector& mined_blocks, MinedBlockStats stats) +{ + const auto stale_begin{std::find_if(mined_blocks.begin(), mined_blocks.end(), [&](const MinedBlockStats& block) { + return block.m_height >= stats.m_height; + })}; + const auto stale_count{std::distance(stale_begin, mined_blocks.end())}; + if (stale_count > 0) { + LogDebug(BCLog::ESTIMATEFEE, + "%s: connected block height=%s discards tracked mined-block stats " + "from height=%s to height=%s; stale_stats=%s", + FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY), + stats.m_height, + stale_begin->m_height, + mined_blocks.back().m_height, + stale_count); + } + mined_blocks.erase(stale_begin, mined_blocks.end()); + if (!mined_blocks.empty() && mined_blocks.back().m_height + 1 != stats.m_height) { + LogDebug(BCLog::ESTIMATEFEE, + "%s: clearing mined-block stats after height gap; tracked_stats=%s " + "expected_height=%s received_height=%s", + FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY), + mined_blocks.size(), + mined_blocks.back().m_height + 1, + stats.m_height); + mined_blocks.clear(); + } + + if (mined_blocks.size() == MEMPOOL_HEALTH_WINDOW_BLOCKS) mined_blocks.erase(mined_blocks.begin()); + mined_blocks.push_back(stats); +} +} // namespace + MemPoolFeeRateEstimator::Percentiles MemPoolFeeRateEstimator::CalculateMaxWeightPercentiles(std::span chunk_feerates) { Assume(std::is_sorted(chunk_feerates.begin(), chunk_feerates.end(), [](const auto& a, const auto& b) { return ByRatio{a} > ByRatio{b}; })); @@ -59,18 +98,107 @@ void MemPoolFeeRateEstimatorCache::Update(FeePerVSize conservative, FeePerVSize m_last_updated = NodeClock::now(); } +void MemPoolFeeRateEstimatorCache::Clear() +{ + m_fee_rate_estimation.reset(); + m_tip_hash.SetNull(); + m_last_updated = {}; +} + //! Build the error result for a failed mempool fee rate estimation. static util::Unexpected EstimationError(std::string error) { return EstimationError(FeeRateEstimatorType::MEMPOOL_POLICY, MEMPOOL_FEE_ESTIMATOR_MAX_TARGET, std::move(error)); } +static std::optional MempoolHealthError(MemPoolFeeRateEstimator::MempoolHealth health) +{ + switch (health) { + case MemPoolFeeRateEstimator::MempoolHealth::INSUFFICIENT_DATA: + return "Not enough recent block data for fee rate estimation"; + case MemPoolFeeRateEstimator::MempoolHealth::LOW_COVERAGE: + return "Mempool is unreliable for fee rate estimation"; + case MemPoolFeeRateEstimator::MempoolHealth::HEALTHY: + return std::nullopt; + } + Assume(false); + return std::nullopt; +} + +void MemPoolFeeRateEstimator::MempoolTxsRemovedForBlock(const std::shared_ptr& block, + const std::vector& txs_removed_for_block, + unsigned int block_height) +{ + LOCK(cs); + Assert(!block->vtx.empty()); + // Accumulate total block weight and removed mempool tx weight, both excluding the coinbase. + const auto get_tx_weight = [](const CTransactionRef& tx) { + return static_cast(GetTransactionWeight(*tx)); + }; + // Skip vtx[0], which is the coinbase. + const uint64_t block_weight = std::accumulate(std::next(block->vtx.begin()), block->vtx.end(), uint64_t{0}, + [&](uint64_t acc, const CTransactionRef& tx) { + return acc + get_tx_weight(tx); + }); + const uint64_t removed_weight = std::accumulate( + txs_removed_for_block.begin(), txs_removed_for_block.end(), uint64_t{0}, + [&](uint64_t acc, const RemovedMempoolTransactionInfo& tx) { + return acc + get_tx_weight(tx.info.m_tx); + }); + AddMinedBlockStats(m_prev_mined_blocks, {block_height, removed_weight, block_weight}); + m_cache.Clear(); +} + +// Require at least one block worth of activity across the window before using +// the coverage ratio as a representative mempool health signal. +static constexpr uint64_t MIN_REPRESENTATIVE_WINDOW_WEIGHT{DEFAULT_BLOCK_MAX_WEIGHT}; + +MemPoolFeeRateEstimator::MempoolHealth MemPoolFeeRateEstimator::GetMempoolHealth() const +{ + LOCK(cs); + const auto estimator_name{FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY)}; + if (m_prev_mined_blocks.size() < MEMPOOL_HEALTH_WINDOW_BLOCKS) { + LogDebug(BCLog::ESTIMATEFEE, "%s: mempool health check failed; tracked_blocks=%s required_blocks=%s", + estimator_name, m_prev_mined_blocks.size(), MEMPOOL_HEALTH_WINDOW_BLOCKS); + return MempoolHealth::INSUFFICIENT_DATA; + } + uint64_t total_block_weight{0}; + uint64_t total_removed_weight{0}; + uint64_t expected_height{m_prev_mined_blocks.front().m_height}; + for (const auto& block : m_prev_mined_blocks) { + Assume(block.m_height == expected_height); + ++expected_height; + total_block_weight += block.m_block_weight; + total_removed_weight += block.m_removed_block_txs_weight; + } + // Too little block activity for the coverage ratio to be meaningful; skip it. + if (total_block_weight < MIN_REPRESENTATIVE_WINDOW_WEIGHT) { + LogDebug(BCLog::ESTIMATEFEE, "%s: mempool health check passed; low activity, total_block_weight=%s minimum=%s", + estimator_name, total_block_weight, MIN_REPRESENTATIVE_WINDOW_WEIGHT); + return MempoolHealth::HEALTHY; + } + const double representation_ratio = static_cast(total_removed_weight) / total_block_weight; + LogDebug(BCLog::ESTIMATEFEE, + "%s: mempool health check %s; removed_weight=%s total_block_weight=%s " + "coverage=%.2f required_coverage=%.2f", + estimator_name, + representation_ratio >= MEMPOOL_REPRESENTATION_THRESHOLD ? "passed" : "failed", + total_removed_weight, + total_block_weight, + representation_ratio, + MEMPOOL_REPRESENTATION_THRESHOLD); + return representation_ratio >= MEMPOOL_REPRESENTATION_THRESHOLD ? MempoolHealth::HEALTHY : MempoolHealth::LOW_COVERAGE; +} + util::Expected MemPoolFeeRateEstimator::EstimateFeeRate(bool conservative) const { constexpr auto estimator_type{FeeRateEstimatorType::MEMPOOL_POLICY}; if (!m_mempool.GetLoadTried()) { return EstimationError(strprintf("%s: Mempool not loaded yet, no fee rate estimate available", FeeRateEstimatorTypeToString(estimator_type))); } + if (auto error{MempoolHealthError(GetMempoolHealth())}) { + return EstimationError(strprintf("%s: %s", FeeRateEstimatorTypeToString(estimator_type), *error)); + } // The estimator lock is not held while building a block template, so // in a rare edge case concurrent callers may duplicate work. // diff --git a/src/policy/fees/mempool_estimator.h b/src/policy/fees/mempool_estimator.h index 82a90600aa5..73d0e1375d7 100644 --- a/src/policy/fees/mempool_estimator.h +++ b/src/policy/fees/mempool_estimator.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_POLICY_FEES_MEMPOOL_ESTIMATOR_H #define BITCOIN_POLICY_FEES_MEMPOOL_ESTIMATOR_H +#include #include #include #include @@ -14,17 +15,36 @@ #include #include +#include #include #include +#include +class CBlock; class ChainstateManager; class CTxMemPool; +struct RemovedMempoolTransactionInfo; + // Fee rate estimate for confirmation target above this is not reliable, // as mempool conditions are likely to change. constexpr int MEMPOOL_FEE_ESTIMATOR_MAX_TARGET{2}; constexpr std::chrono::seconds CACHE_LIFE{7}; +// Constants for mempool sanity checks. +constexpr size_t MEMPOOL_HEALTH_WINDOW_BLOCKS = 6; +constexpr double MEMPOOL_REPRESENTATION_THRESHOLD = 0.75; + +//! Weight statistics for a recently mined block, used to assess mempool coverage. +struct MinedBlockStats { + //! Block height. + uint64_t m_height{0}; + //! Weight of mempool transactions removed for this block (excluding coinbase). + uint64_t m_removed_block_txs_weight{0}; + //! Total non-coinbase transaction weight in the block. + uint64_t m_block_weight{0}; +}; + /** * MemPoolFeeRateEstimatorCache holds a cache of recent fee rate estimates. * A cached fee rate is only provided while it is not older than CACHE_LIFE @@ -46,6 +66,8 @@ public: std::optional GetCachedEstimate(const uint256& tip_hash) const; /** Update the cache with new estimates computed on tip_hash. */ void Update(FeePerVSize conservative, FeePerVSize economical, const uint256& tip_hash); + /** Clear cached fee rate estimates. */ + void Clear(); private: std::optional m_fee_rate_estimation; @@ -86,7 +108,27 @@ public: return MEMPOOL_FEE_ESTIMATOR_MAX_TARGET; } + void MempoolTxsRemovedForBlock(const std::shared_ptr& block, + const std::vector& txs_removed_for_block, + unsigned int block_height) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + //! Health of the recent mined-block window for fee rate estimation. + enum class MempoolHealth { + //! Recent blocks represent the mempool well enough to estimate a fee rate. + HEALTHY, + //! Too few recent mined blocks to estimate a fee rate. + INSUFFICIENT_DATA, + //! Recent blocks include too few mempool transactions to estimate a fee rate. + LOW_COVERAGE, + }; + MempoolHealth GetMempoolHealth() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + //! Checks if recent mined blocks indicate a healthy mempool state. + bool IsMempoolHealthy() const EXCLUSIVE_LOCKS_REQUIRED(!cs) { return GetMempoolHealth() == MempoolHealth::HEALTHY; } + private: + //! Tracks weight statistics for the last MEMPOOL_HEALTH_WINDOW_BLOCKS mined blocks. + std::vector m_prev_mined_blocks GUARDED_BY(cs); + const CTxMemPool& m_mempool; ChainstateManager& m_chainman; mutable Mutex cs; diff --git a/src/test/mempool_fee_estimator_tests.cpp b/src/test/mempool_fee_estimator_tests.cpp index 2331ae7c6b7..cb31fb48b36 100644 --- a/src/test/mempool_fee_estimator_tests.cpp +++ b/src/test/mempool_fee_estimator_tests.cpp @@ -3,6 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include +#include #include #include #include @@ -33,6 +35,31 @@ static inline CTransactionRef MakeRandomTx() return MakeTransactionRef(tx); } +void AddRemovedBlock(MemPoolFeeRateEstimator& fee_est, + int32_t removed_txs_weight, + int32_t block_txs_weight, + unsigned int& height) +{ + auto block = std::make_shared(); + std::vector removed_txs; + TestMemPoolEntryHelper entry; + Assert(block_txs_weight >= removed_txs_weight); + block->vtx.emplace_back(MakeRandomTx()); // Add a coinbase tx + while (block_txs_weight > 0) { + auto tx = MakeRandomTx(); + auto tx_weight = GetTransactionWeight(*tx); + if (block_txs_weight - tx_weight < 0) break; + block->vtx.emplace_back(tx); + block_txs_weight -= tx_weight; + if (removed_txs_weight - tx_weight >= 0) { + removed_txs.emplace_back(entry.FromTx(tx)); + removed_txs_weight -= tx_weight; + } + } + fee_est.MempoolTxsRemovedForBlock(block, removed_txs, height); + height += 1; +} + BOOST_AUTO_TEST_CASE(calculate_max_weight_percentiles) { // With no chunks neither percentile can be populated. @@ -112,6 +139,102 @@ BOOST_AUTO_TEST_CASE(MempoolFeeRateEstimator) BOOST_CHECK_EQUAL(result.error().reason, unloaded_err); } m_node.mempool->SetLoadTried(true); + + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + BOOST_CHECK(mempool_estimator.GetMempoolHealth() == MemPoolFeeRateEstimator::MempoolHealth::INSUFFICIENT_DATA); + { + const auto result = mempool_estimator.EstimateFeeRate(/*conservative=*/true); + const std::string insufficient_err{strprintf("%s: Not enough recent block data for fee rate estimation", + FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY))}; + BOOST_CHECK(!result); + BOOST_CHECK_EQUAL(result.error().reason, insufficient_err); + } + { + MemPoolFeeRateEstimator custom_mempool_estimator{*m_node.mempool, *m_node.chainman}; + unsigned int custom_height{100}; + for (size_t block_count{1}; block_count < MEMPOOL_HEALTH_WINDOW_BLOCKS; ++block_count) { + AddRemovedBlock(custom_mempool_estimator, + /*removed_txs_weight=*/0, + /*block_txs_weight=*/0, + custom_height); + BOOST_CHECK(!custom_mempool_estimator.IsMempoolHealthy()); + } + { + const int64_t low_activity_weight{1000}; + AddRemovedBlock(custom_mempool_estimator, low_activity_weight / 2, low_activity_weight, custom_height); + } + // Below one block worth of total activity across the full window, even + // poor coverage in the only non-empty block is too noisy to reject the + // mempool as unhealthy. + BOOST_CHECK(custom_mempool_estimator.IsMempoolHealthy()); + } + size_t block_count = 1; + const int64_t weight{DEFAULT_BLOCK_MAX_WEIGHT / 2}; + unsigned int height = 100; + // Equal weight + while (block_count <= MEMPOOL_HEALTH_WINDOW_BLOCKS) { + AddRemovedBlock(mempool_estimator, weight, weight, height); + if (block_count < MEMPOOL_HEALTH_WINDOW_BLOCKS) { + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + } + block_count += 1; + } + // Total txs weight ~11999k WU (~3.0 blocks), removed txs ~11999k WU (~3.0 blocks); coverage = 100%. + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + // Adding a single underrepresented block will not make the mempool unhealthy + // while the window coverage remains above the threshold. + AddRemovedBlock(mempool_estimator, weight / 2, weight, height); + // Total txs weight ~11999k WU (~3.0 blocks), removed txs ~10999k WU (~2.75 blocks); coverage = ~92%. + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + // Empty block + // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~8999k WU (~2.25 blocks); coverage = 90%. + AddRemovedBlock(mempool_estimator, 0, 0, height); + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~7999k WU (~2.0 blocks); coverage = 80%. + AddRemovedBlock(mempool_estimator, weight / 2, weight, height); + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~7000k WU (~1.75 blocks); coverage = 70%. + AddRemovedBlock(mempool_estimator, weight / 2, weight, height); + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + block_count = 1; + while (block_count <= 3) { + AddRemovedBlock(mempool_estimator, weight, weight, height); + if (block_count < 3) { + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + } + block_count += 1; + } + // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~7999k WU (~2.0 blocks); coverage = 80%. + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + + // Reorg out and replace the last block. Replacing the tip block should keep a full + // healthy window when the replacement block has good mempool representation. + height -= 1; + AddRemovedBlock(mempool_estimator, weight, weight, height); + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + + // Reorg out the last two blocks. The estimator should discard the stale suffix, + // become temporarily unhealthy due to having fewer than MEMPOOL_HEALTH_WINDOW_BLOCKS stats, + // then recover after the replacement chain catches up. + height -= 2; + AddRemovedBlock(mempool_estimator, weight, weight, height); + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + AddRemovedBlock(mempool_estimator, weight, weight, height); + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); + + // A forward height gap (e.g. stale persisted stats after an unclean shutdown + // while the chain advanced) resets the tracked window entirely; the estimator + // stays unhealthy until a full window of contiguous blocks is seen again. + height += 3; + AddRemovedBlock(mempool_estimator, weight, weight, height); + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + for (size_t i = 1; i < MEMPOOL_HEALTH_WINDOW_BLOCKS; ++i) { + AddRemovedBlock(mempool_estimator, weight, weight, height); + if (i < MEMPOOL_HEALTH_WINDOW_BLOCKS - 1) { + BOOST_CHECK(!mempool_estimator.IsMempoolHealthy()); + } + } + BOOST_CHECK(mempool_estimator.IsMempoolHealthy()); { LOCK(m_node.mempool->cs); BOOST_CHECK_EQUAL(m_node.mempool->GetTotalTxSize(), 0);