fees: gate mempool estimates on recent block coverage

Gate the mempool fee rate estimator on a coverage check: recent
connected blocks must be well represented by transactions removed from
our mempool.

Track per-block weight for the last MEMPOOL_HEALTH_WINDOW_BLOCKS blocks.
AddMinedBlockStats drops stats at or above a connected block's height
before appending it, and resets the window on a forward height gap so
tracked heights stay consecutive.

Only apply the coverage ratio once the window holds at least one block
of transactions; below that activity is too low for the ratio to be
meaningful, so treat the mempool as healthy.

Replace a boolean health check with a MempoolHealth enum so
EstimateFeeRate() can report whether estimation is unavailable because
too few recent blocks have been tracked (INSUFFICIENT_DATA) or because
recent blocks poorly represent the mempool (LOW_COVERAGE).

src/policy/fees/estimator_man.cpp

@@ -72,9 +72,10 @@ void FeeRateEstimatorManager::TransactionRemovedFromMempool(const CTransactionRe
     m_block_policy_estimator->removeTx(tx->GetHash());
 }
 
-void FeeRateEstimatorManager::MempoolTransactionsRemovedForBlock(const std::shared_ptr<const CBlock>& /*block*/, const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int block_height)
+void FeeRateEstimatorManager::MempoolTransactionsRemovedForBlock(const std::shared_ptr<const CBlock>& block, const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int block_height)
 {
     m_block_policy_estimator->processBlock(txs_removed_for_block, block_height);
+    m_mempool_estimator->MempoolTxsRemovedForBlock(block, txs_removed_for_block, block_height);
 }
 
 CFeeRate FeeRateEstimatorManager::BlockPolicyEstimateRawFee(unsigned int target, double threshold, FeeEstimateHorizon horizon, EstimationResult* buckets) const

src/policy/fees/mempool_estimator.cpp

@@ -17,8 +17,47 @@
 #include <validation.h>
 
 #include <algorithm>
+#include <iterator>
+#include <numeric>
+#include <optional>
+#include <string>
+#include <string_view>
 #include <utility>
 
+namespace {
+void AddMinedBlockStats(std::vector<MinedBlockStats>& mined_blocks, MinedBlockStats stats)
+{
+    const auto stale_begin{std::find_if(mined_blocks.begin(), mined_blocks.end(), [&](const MinedBlockStats& block) {
+        return block.m_height >= stats.m_height;
+    })};
+    const auto stale_count{std::distance(stale_begin, mined_blocks.end())};
+    if (stale_count > 0) {
+        LogDebug(BCLog::ESTIMATEFEE,
+                 "%s: connected block height=%s discards tracked mined-block stats "
+                 "from height=%s to height=%s; stale_stats=%s",
+                 FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY),
+                 stats.m_height,
+                 stale_begin->m_height,
+                 mined_blocks.back().m_height,
+                 stale_count);
+    }
+    mined_blocks.erase(stale_begin, mined_blocks.end());
+    if (!mined_blocks.empty() && mined_blocks.back().m_height + 1 != stats.m_height) {
+        LogDebug(BCLog::ESTIMATEFEE,
+                 "%s: clearing mined-block stats after height gap; tracked_stats=%s "
+                 "expected_height=%s received_height=%s",
+                 FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY),
+                 mined_blocks.size(),
+                 mined_blocks.back().m_height + 1,
+                 stats.m_height);
+        mined_blocks.clear();
+    }
+
+    if (mined_blocks.size() == MEMPOOL_HEALTH_WINDOW_BLOCKS) mined_blocks.erase(mined_blocks.begin());
+    mined_blocks.push_back(stats);
+}
+} // namespace
+
 MemPoolFeeRateEstimator::Percentiles MemPoolFeeRateEstimator::CalculateMaxWeightPercentiles(std::span<const FeePerVSize> chunk_feerates)
 {
     Assume(std::is_sorted(chunk_feerates.begin(), chunk_feerates.end(), [](const auto& a, const auto& b) { return ByRatio{a} > ByRatio{b}; }));
@@ -59,18 +98,107 @@ void MemPoolFeeRateEstimatorCache::Update(FeePerVSize conservative, FeePerVSize
     m_last_updated = NodeClock::now();
 }
 
+void MemPoolFeeRateEstimatorCache::Clear()
+{
+    m_fee_rate_estimation.reset();
+    m_tip_hash.SetNull();
+    m_last_updated = {};
+}
+
 //! Build the error result for a failed mempool fee rate estimation.
 static util::Unexpected<FeeRateEstimationError> EstimationError(std::string error)
 {
     return EstimationError(FeeRateEstimatorType::MEMPOOL_POLICY, MEMPOOL_FEE_ESTIMATOR_MAX_TARGET, std::move(error));
 }
 
+static std::optional<std::string_view> MempoolHealthError(MemPoolFeeRateEstimator::MempoolHealth health)
+{
+    switch (health) {
+    case MemPoolFeeRateEstimator::MempoolHealth::INSUFFICIENT_DATA:
+        return "Not enough recent block data for fee rate estimation";
+    case MemPoolFeeRateEstimator::MempoolHealth::LOW_COVERAGE:
+        return "Mempool is unreliable for fee rate estimation";
+    case MemPoolFeeRateEstimator::MempoolHealth::HEALTHY:
+        return std::nullopt;
+    }
+    Assume(false);
+    return std::nullopt;
+}
+
+void MemPoolFeeRateEstimator::MempoolTxsRemovedForBlock(const std::shared_ptr<const CBlock>& block,
+                                                        const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block,
+                                                        unsigned int block_height)
+{
+    LOCK(cs);
+    Assert(!block->vtx.empty());
+    // Accumulate total block weight and removed mempool tx weight, both excluding the coinbase.
+    const auto get_tx_weight = [](const CTransactionRef& tx) {
+        return static_cast<uint64_t>(GetTransactionWeight(*tx));
+    };
+    // Skip vtx[0], which is the coinbase.
+    const uint64_t block_weight = std::accumulate(std::next(block->vtx.begin()), block->vtx.end(), uint64_t{0},
+                                                  [&](uint64_t acc, const CTransactionRef& tx) {
+                                                      return acc + get_tx_weight(tx);
+                                                  });
+    const uint64_t removed_weight = std::accumulate(
+        txs_removed_for_block.begin(), txs_removed_for_block.end(), uint64_t{0},
+        [&](uint64_t acc, const RemovedMempoolTransactionInfo& tx) {
+            return acc + get_tx_weight(tx.info.m_tx);
+        });
+    AddMinedBlockStats(m_prev_mined_blocks, {block_height, removed_weight, block_weight});
+    m_cache.Clear();
+}
+
+// Require at least one block worth of activity across the window before using
+// the coverage ratio as a representative mempool health signal.
+static constexpr uint64_t MIN_REPRESENTATIVE_WINDOW_WEIGHT{DEFAULT_BLOCK_MAX_WEIGHT};
+
+MemPoolFeeRateEstimator::MempoolHealth MemPoolFeeRateEstimator::GetMempoolHealth() const
+{
+    LOCK(cs);
+    const auto estimator_name{FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY)};
+    if (m_prev_mined_blocks.size() < MEMPOOL_HEALTH_WINDOW_BLOCKS) {
+        LogDebug(BCLog::ESTIMATEFEE, "%s: mempool health check failed; tracked_blocks=%s required_blocks=%s",
+                 estimator_name, m_prev_mined_blocks.size(), MEMPOOL_HEALTH_WINDOW_BLOCKS);
+        return MempoolHealth::INSUFFICIENT_DATA;
+    }
+    uint64_t total_block_weight{0};
+    uint64_t total_removed_weight{0};
+    uint64_t expected_height{m_prev_mined_blocks.front().m_height};
+    for (const auto& block : m_prev_mined_blocks) {
+        Assume(block.m_height == expected_height);
+        ++expected_height;
+        total_block_weight += block.m_block_weight;
+        total_removed_weight += block.m_removed_block_txs_weight;
+    }
+    // Too little block activity for the coverage ratio to be meaningful; skip it.
+    if (total_block_weight < MIN_REPRESENTATIVE_WINDOW_WEIGHT) {
+        LogDebug(BCLog::ESTIMATEFEE, "%s: mempool health check passed; low activity, total_block_weight=%s minimum=%s",
+                 estimator_name, total_block_weight, MIN_REPRESENTATIVE_WINDOW_WEIGHT);
+        return MempoolHealth::HEALTHY;
+    }
+    const double representation_ratio = static_cast<double>(total_removed_weight) / total_block_weight;
+    LogDebug(BCLog::ESTIMATEFEE,
+             "%s: mempool health check %s; removed_weight=%s total_block_weight=%s "
+             "coverage=%.2f required_coverage=%.2f",
+             estimator_name,
+             representation_ratio >= MEMPOOL_REPRESENTATION_THRESHOLD ? "passed" : "failed",
+             total_removed_weight,
+             total_block_weight,
+             representation_ratio,
+             MEMPOOL_REPRESENTATION_THRESHOLD);
+    return representation_ratio >= MEMPOOL_REPRESENTATION_THRESHOLD ? MempoolHealth::HEALTHY : MempoolHealth::LOW_COVERAGE;
+}
+
 util::Expected<FeeRateEstimation, FeeRateEstimationError> MemPoolFeeRateEstimator::EstimateFeeRate(bool conservative) const
 {
     constexpr auto estimator_type{FeeRateEstimatorType::MEMPOOL_POLICY};
     if (!m_mempool.GetLoadTried()) {
         return EstimationError(strprintf("%s: Mempool not loaded yet, no fee rate estimate available", FeeRateEstimatorTypeToString(estimator_type)));
     }
+    if (auto error{MempoolHealthError(GetMempoolHealth())}) {
+        return EstimationError(strprintf("%s: %s", FeeRateEstimatorTypeToString(estimator_type), *error));
+    }
     // The estimator lock is not held while building a block template, so
     // in a rare edge case concurrent callers may duplicate work.
     //

src/policy/fees/mempool_estimator.h

@@ -5,6 +5,7 @@
 #ifndef BITCOIN_POLICY_FEES_MEMPOOL_ESTIMATOR_H
 #define BITCOIN_POLICY_FEES_MEMPOOL_ESTIMATOR_H
 
+#include <primitives/transaction.h>
 #include <sync.h>
 #include <threadsafety.h>
 #include <uint256.h>
@@ -14,17 +15,36 @@
 #include <util/time.h>
 
 #include <chrono>
+#include <memory>
 #include <optional>
 #include <span>
+#include <vector>
 
+class CBlock;
 class ChainstateManager;
 class CTxMemPool;
 
+struct RemovedMempoolTransactionInfo;
+
 // Fee rate estimate for confirmation target above this is not reliable,
 // as mempool conditions are likely to change.
 constexpr int MEMPOOL_FEE_ESTIMATOR_MAX_TARGET{2};
 constexpr std::chrono::seconds CACHE_LIFE{7};
 
+// Constants for mempool sanity checks.
+constexpr size_t MEMPOOL_HEALTH_WINDOW_BLOCKS = 6;
+constexpr double MEMPOOL_REPRESENTATION_THRESHOLD = 0.75;
+
+//! Weight statistics for a recently mined block, used to assess mempool coverage.
+struct MinedBlockStats {
+    //! Block height.
+    uint64_t m_height{0};
+    //! Weight of mempool transactions removed for this block (excluding coinbase).
+    uint64_t m_removed_block_txs_weight{0};
+    //! Total non-coinbase transaction weight in the block.
+    uint64_t m_block_weight{0};
+};
+
 /**
  * MemPoolFeeRateEstimatorCache holds a cache of recent fee rate estimates.
  * A cached fee rate is only provided while it is not older than CACHE_LIFE
@@ -46,6 +66,8 @@ public:
     std::optional<FeeRateEstimate> GetCachedEstimate(const uint256& tip_hash) const;
     /** Update the cache with new estimates computed on tip_hash. */
     void Update(FeePerVSize conservative, FeePerVSize economical, const uint256& tip_hash);
+    /** Clear cached fee rate estimates. */
+    void Clear();
 
 private:
     std::optional<FeeRateEstimate> m_fee_rate_estimation;
@@ -86,7 +108,27 @@ public:
         return MEMPOOL_FEE_ESTIMATOR_MAX_TARGET;
     }
 
+    void MempoolTxsRemovedForBlock(const std::shared_ptr<const CBlock>& block,
+                                   const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block,
+                                   unsigned int block_height)
+        EXCLUSIVE_LOCKS_REQUIRED(!cs);
+    //! Health of the recent mined-block window for fee rate estimation.
+    enum class MempoolHealth {
+        //! Recent blocks represent the mempool well enough to estimate a fee rate.
+        HEALTHY,
+        //! Too few recent mined blocks to estimate a fee rate.
+        INSUFFICIENT_DATA,
+        //! Recent blocks include too few mempool transactions to estimate a fee rate.
+        LOW_COVERAGE,
+    };
+    MempoolHealth GetMempoolHealth() const EXCLUSIVE_LOCKS_REQUIRED(!cs);
+    //! Checks if recent mined blocks indicate a healthy mempool state.
+    bool IsMempoolHealthy() const EXCLUSIVE_LOCKS_REQUIRED(!cs) { return GetMempoolHealth() == MempoolHealth::HEALTHY; }
+
 private:
+    //! Tracks weight statistics for the last MEMPOOL_HEALTH_WINDOW_BLOCKS mined blocks.
+    std::vector<MinedBlockStats> m_prev_mined_blocks GUARDED_BY(cs);
+
     const CTxMemPool& m_mempool;
     ChainstateManager& m_chainman;
     mutable Mutex cs;

src/test/mempool_fee_estimator_tests.cpp

@@ -3,6 +3,8 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include <policy/fees/mempool_estimator.h>
+#include <policy/policy.h>
+#include <primitives/block.h>
 #include <random.h>
 #include <test/util/setup_common.h>
 #include <test/util/txmempool.h>
@@ -33,6 +35,31 @@ static inline CTransactionRef MakeRandomTx()
     return MakeTransactionRef(tx);
 }
 
+void AddRemovedBlock(MemPoolFeeRateEstimator& fee_est,
+                     int32_t removed_txs_weight,
+                     int32_t block_txs_weight,
+                     unsigned int& height)
+{
+    auto block = std::make_shared<CBlock>();
+    std::vector<RemovedMempoolTransactionInfo> removed_txs;
+    TestMemPoolEntryHelper entry;
+    Assert(block_txs_weight >= removed_txs_weight);
+    block->vtx.emplace_back(MakeRandomTx()); // Add a coinbase tx
+    while (block_txs_weight > 0) {
+        auto tx = MakeRandomTx();
+        auto tx_weight = GetTransactionWeight(*tx);
+        if (block_txs_weight - tx_weight < 0) break;
+        block->vtx.emplace_back(tx);
+        block_txs_weight -= tx_weight;
+        if (removed_txs_weight - tx_weight >= 0) {
+            removed_txs.emplace_back(entry.FromTx(tx));
+            removed_txs_weight -= tx_weight;
+        }
+    }
+    fee_est.MempoolTxsRemovedForBlock(block, removed_txs, height);
+    height += 1;
+}
+
 BOOST_AUTO_TEST_CASE(calculate_max_weight_percentiles)
 {
     // With no chunks neither percentile can be populated.
@@ -112,6 +139,102 @@ BOOST_AUTO_TEST_CASE(MempoolFeeRateEstimator)
         BOOST_CHECK_EQUAL(result.error().reason, unloaded_err);
     }
     m_node.mempool->SetLoadTried(true);
+
+    BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+    BOOST_CHECK(mempool_estimator.GetMempoolHealth() == MemPoolFeeRateEstimator::MempoolHealth::INSUFFICIENT_DATA);
+    {
+        const auto result = mempool_estimator.EstimateFeeRate(/*conservative=*/true);
+        const std::string insufficient_err{strprintf("%s: Not enough recent block data for fee rate estimation",
+                                                     FeeRateEstimatorTypeToString(FeeRateEstimatorType::MEMPOOL_POLICY))};
+        BOOST_CHECK(!result);
+        BOOST_CHECK_EQUAL(result.error().reason, insufficient_err);
+    }
+    {
+        MemPoolFeeRateEstimator custom_mempool_estimator{*m_node.mempool, *m_node.chainman};
+        unsigned int custom_height{100};
+        for (size_t block_count{1}; block_count < MEMPOOL_HEALTH_WINDOW_BLOCKS; ++block_count) {
+            AddRemovedBlock(custom_mempool_estimator,
+                            /*removed_txs_weight=*/0,
+                            /*block_txs_weight=*/0,
+                            custom_height);
+            BOOST_CHECK(!custom_mempool_estimator.IsMempoolHealthy());
+        }
+        {
+            const int64_t low_activity_weight{1000};
+            AddRemovedBlock(custom_mempool_estimator, low_activity_weight / 2, low_activity_weight, custom_height);
+        }
+        // Below one block worth of total activity across the full window, even
+        // poor coverage in the only non-empty block is too noisy to reject the
+        // mempool as unhealthy.
+        BOOST_CHECK(custom_mempool_estimator.IsMempoolHealthy());
+    }
+    size_t block_count = 1;
+    const int64_t weight{DEFAULT_BLOCK_MAX_WEIGHT / 2};
+    unsigned int height = 100;
+    // Equal weight
+    while (block_count <= MEMPOOL_HEALTH_WINDOW_BLOCKS) {
+        AddRemovedBlock(mempool_estimator, weight, weight, height);
+        if (block_count < MEMPOOL_HEALTH_WINDOW_BLOCKS) {
+            BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+        }
+        block_count += 1;
+    }
+    // Total txs weight ~11999k WU (~3.0 blocks), removed txs ~11999k WU (~3.0 blocks); coverage = 100%.
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+    // Adding a single underrepresented block will not make the mempool unhealthy
+    // while the window coverage remains above the threshold.
+    AddRemovedBlock(mempool_estimator, weight / 2, weight, height);
+    // Total txs weight ~11999k WU (~3.0 blocks), removed txs ~10999k WU (~2.75 blocks); coverage = ~92%.
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+    // Empty block
+    // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~8999k WU (~2.25 blocks); coverage = 90%.
+    AddRemovedBlock(mempool_estimator, 0, 0, height);
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+    // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~7999k WU (~2.0 blocks); coverage = 80%.
+    AddRemovedBlock(mempool_estimator, weight / 2, weight, height);
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+    // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~7000k WU (~1.75 blocks); coverage = 70%.
+    AddRemovedBlock(mempool_estimator, weight / 2, weight, height);
+    BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+    block_count = 1;
+    while (block_count <= 3) {
+        AddRemovedBlock(mempool_estimator, weight, weight, height);
+        if (block_count < 3) {
+            BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+        }
+        block_count += 1;
+    }
+    // Total txs weight ~9999k WU (~2.5 blocks), removed txs ~7999k WU (~2.0 blocks); coverage = 80%.
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+
+    // Reorg out and replace the last block. Replacing the tip block should keep a full
+    // healthy window when the replacement block has good mempool representation.
+    height -= 1;
+    AddRemovedBlock(mempool_estimator, weight, weight, height);
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+
+    // Reorg out the last two blocks. The estimator should discard the stale suffix,
+    // become temporarily unhealthy due to having fewer than MEMPOOL_HEALTH_WINDOW_BLOCKS stats,
+    // then recover after the replacement chain catches up.
+    height -= 2;
+    AddRemovedBlock(mempool_estimator, weight, weight, height);
+    BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+    AddRemovedBlock(mempool_estimator, weight, weight, height);
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
+
+    // A forward height gap (e.g. stale persisted stats after an unclean shutdown
+    // while the chain advanced) resets the tracked window entirely; the estimator
+    // stays unhealthy until a full window of contiguous blocks is seen again.
+    height += 3;
+    AddRemovedBlock(mempool_estimator, weight, weight, height);
+    BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+    for (size_t i = 1; i < MEMPOOL_HEALTH_WINDOW_BLOCKS; ++i) {
+        AddRemovedBlock(mempool_estimator, weight, weight, height);
+        if (i < MEMPOOL_HEALTH_WINDOW_BLOCKS - 1) {
+            BOOST_CHECK(!mempool_estimator.IsMempoolHealthy());
+        }
+    }
+    BOOST_CHECK(mempool_estimator.IsMempoolHealthy());
     {
         LOCK(m_node.mempool->cs);
         BOOST_CHECK_EQUAL(m_node.mempool->GetTotalTxSize(), 0);