Merge bitcoin/bitcoin#28530: tests, bug fix: DisconnectedBlockTransactions rewrite followups

9b3da70bd0 [test] DisconnectedBlockTransactions::DynamicMemoryUsage (glozow)
b2d0447964 bugfix: correct DisconnectedBlockTransactions memory usage (stickies-v)
f4254e2098 assume duplicate transactions are not added to `iters_by_txid` (ismaelsadeeq)
29eb219c12 move only: move implementation code to disconnected_transactions.cpp (ismaelsadeeq)
81dfeddea7 refactor: update `MAX_DISCONNECTED_TX_POOL` from kb to bytes (ismaelsadeeq)

Pull request description:

  This PR is a follow-up to fix review comments and a bugfix from #28385

  The PR

  - Updated `DisconnectedBlockTransactions`'s `MAX_DISCONNECTED_TX_POOL` from kb to bytes.
  - Moved `DisconnectedBlockTransactions` implementation code to `kernel/disconnected_transactions.cpp`.
  - `AddTransactionsFromBlock` now assume duplicate transactions are not passed by asserting after inserting each transaction to `iters_by_txid`.
  - Included a Bug fix: In the current master we are underestimating the memory usage of `DisconnectedBlockTransactions`.

      * When adding and subtracting `cachedInnerUsage` we call `RecursiveDynamicUsage` with `CTransaction` which invokes this [`RecursiveDynamicUsage(const CTransaction& tx)`](6e721c923c/src/core_memusage.h (L32)) version of `RecursiveDynamicUsage`, the output of that call only account for the memory usage of the inputs and outputs of the `CTransaction`, this omits the memory usage of the `CTransaction` object and the control block.
      * This PR fixes this bug by calling `RecursiveDynamicUsage` with `CTransactionRef` when adding and subtracting `cachedInnerUsage` which invokes [`RecursiveDynamicUsage(const std::shared_ptr<X>& p)`](6e721c923c/src/core_memusage.h (L67)) version of `RecursiveDynamicUsage` the output of the calculation accounts for the` CTransaction` object, the control blocks, inputs and outputs memory usage.
      * see  [comment ](https://github.com/bitcoin/bitcoin/pull/28385#discussion_r1322948452)
  - Added test for DisconnectedBlockTransactions memory limit.

ACKs for top commit:
  stickies-v:
    ACK 9b3da70bd0 - nice work!
  BrandonOdiwuor:
    re ACK 9b3da70bd0
  glozow:
    ACK 9b3da70bd0

Tree-SHA512: 69b9595d09f4d0209038f97081d790cea92ccf63efb94e9e372749979fcbe527f7f17a8e454720cedd12021be0c8e11cf99874625d3dafd9ec602b12dbeb4098

src/test/disconnected_transactions.cpp

@@ -0,0 +1,95 @@
+// Copyright (c) 2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+//
+#include <boost/test/unit_test.hpp>
+#include <core_memusage.h>
+#include <kernel/disconnected_transactions.h>
+#include <test/util/setup_common.h>
+
+BOOST_FIXTURE_TEST_SUITE(disconnected_transactions, TestChain100Setup)
+
+//! Tests that DisconnectedBlockTransactions limits its own memory properly
+BOOST_AUTO_TEST_CASE(disconnectpool_memory_limits)
+{
+    // Use the coinbase transactions from TestChain100Setup. It doesn't matter whether these
+    // transactions would realistically be in a block together, they just need distinct txids and
+    // uniform size for this test to work.
+    std::vector<CTransactionRef> block_vtx(m_coinbase_txns);
+    BOOST_CHECK_EQUAL(block_vtx.size(), 100);
+
+    // Roughly estimate sizes to sanity check that DisconnectedBlockTransactions::DynamicMemoryUsage
+    // is within an expected range.
+
+    // Overhead for the hashmap depends on number of buckets
+    std::unordered_map<uint256, CTransaction*, SaltedTxidHasher> temp_map;
+    temp_map.reserve(1);
+    const size_t MAP_1{memusage::DynamicUsage(temp_map)};
+    temp_map.reserve(100);
+    const size_t MAP_100{memusage::DynamicUsage(temp_map)};
+
+    const size_t TX_USAGE{RecursiveDynamicUsage(block_vtx.front())};
+    for (const auto& tx : block_vtx)
+        BOOST_CHECK_EQUAL(RecursiveDynamicUsage(tx), TX_USAGE);
+
+    // Our overall formula is unordered map overhead + usage per entry.
+    // Implementations may vary, but we're trying to guess the usage of data structures.
+    const size_t ENTRY_USAGE_ESTIMATE{
+        TX_USAGE
+        // list entry: 2 pointers (next pointer and prev pointer) + element itself
+        + memusage::MallocUsage((2 * sizeof(void*)) + sizeof(decltype(block_vtx)::value_type))
+        // unordered map: 1 pointer for the hashtable + key and value
+        + memusage::MallocUsage(sizeof(void*) + sizeof(decltype(temp_map)::key_type)
+                                + sizeof(decltype(temp_map)::value_type))};
+
+    // DisconnectedBlockTransactions that's just big enough for 1 transaction.
+    {
+        DisconnectedBlockTransactions disconnectpool{MAP_1 + ENTRY_USAGE_ESTIMATE};
+        // Add just 2 (and not all 100) transactions to keep the unordered map's hashtable overhead
+        // to a minimum and avoid all (instead of all but 1) transactions getting evicted.
+        std::vector<CTransactionRef> two_txns({block_vtx.at(0), block_vtx.at(1)});
+        auto evicted_txns{disconnectpool.AddTransactionsFromBlock(two_txns)};
+        BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= MAP_1 + ENTRY_USAGE_ESTIMATE);
+
+        // Only 1 transaction can be kept
+        BOOST_CHECK_EQUAL(1, evicted_txns.size());
+        // Transactions are added from back to front and eviction is FIFO.
+        BOOST_CHECK_EQUAL(block_vtx.at(1), evicted_txns.front());
+
+        disconnectpool.clear();
+    }
+
+    // DisconnectedBlockTransactions with a comfortable maximum memory usage so that nothing is evicted.
+    // Record usage so we can check size limiting in the next test.
+    size_t usage_full{0};
+    {
+        const size_t USAGE_100_OVERESTIMATE{MAP_100 + ENTRY_USAGE_ESTIMATE * 100};
+        DisconnectedBlockTransactions disconnectpool{USAGE_100_OVERESTIMATE};
+        auto evicted_txns{disconnectpool.AddTransactionsFromBlock(block_vtx)};
+        BOOST_CHECK_EQUAL(evicted_txns.size(), 0);
+        BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= USAGE_100_OVERESTIMATE);
+
+        usage_full = disconnectpool.DynamicMemoryUsage();
+
+        disconnectpool.clear();
+    }
+
+    // DisconnectedBlockTransactions that's just a little too small for all of the transactions.
+    {
+        const size_t MAX_MEMUSAGE_99{usage_full - sizeof(void*)};
+        DisconnectedBlockTransactions disconnectpool{MAX_MEMUSAGE_99};
+        auto evicted_txns{disconnectpool.AddTransactionsFromBlock(block_vtx)};
+        BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= MAX_MEMUSAGE_99);
+
+        // Only 1 transaction needed to be evicted
+        BOOST_CHECK_EQUAL(1, evicted_txns.size());
+
+        // Transactions are added from back to front and eviction is FIFO.
+        // The last transaction of block_vtx should be the first to be evicted.
+        BOOST_CHECK_EQUAL(block_vtx.back(), evicted_txns.front());
+
+        disconnectpool.clear();
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/test/validation_chainstatemanager_tests.cpp

@@ -579,7 +579,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup)
     // it will initialize instead of attempting to complete validation.
     //
     // Note that this is not a realistic use of DisconnectTip().
-    DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000};
+    DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_BYTES};
     BlockValidationState unused_state;
     {
         LOCK2(::cs_main, bg_chainstate.MempoolMutex());