Merge bitcoin/bitcoin#24539: Add a "tx output spender" index

0b96b9c600 Minimize mempool lock, sync txo spender index only when and if needed (sstone)
3d82ec5bdd Add a "tx output spender" index (sstone)

Pull request description:

  This PR adds a new "tx output spender" index, which allows users to query which tx spent a given outpoint with the `gettxspendingprevout` RPC call that was added by https://github.com/bitcoin/bitcoin/pull/24408.

  Such an index would be extremely useful for Lightning, and probably for most layer-2 protocols that rely on chains of unpublished transactions.

  UPDATE: this PR is ready for review and issues have been addressed:
  - using a watch-only wallet instead would not work if there is a significant number of outpoints to watch (see https://github.com/bitcoin/bitcoin/pull/24539#issuecomment-1276595646)
  - this PR does not require `-txindex` anymore

  We use a composite key with 2 parts (suggested by romanz): hash(spent outpoint) and tx position, with an empty value. Average composite key size is 15 bytes.

  The spending tx can optionally be returned by `gettxspendingprevout` (even it `-txindex is not set`).

ACKs for top commit:
  hodlinator:
    re-ACK 0b96b9c600
  sedited:
    Re-ACK 0b96b9c600
  fjahr:
    ACK 0b96b9c600
  w0xlt:
    reACK 0b96b9c600

Tree-SHA512: 95c2c313ef4086e7d5bf1cf1a3c7b91cfe2bb1a0dcb4c9d3aa8a6e5bfde66aaca48d85a1f1251a780523c3e4356ec8a97fe6f5c7145bc6ccb6f820b26716ae01

src/test/CMakeLists.txt

@@ -116,6 +116,7 @@ add_executable(test_bitcoin
   txdownload_tests.cpp
   txgraph_tests.cpp
   txindex_tests.cpp
+  txospenderindex_tests.cpp
   txpackage_tests.cpp
   txreconciliation_tests.cpp
   txrequest_tests.cpp

src/test/txospenderindex_tests.cpp

@@ -0,0 +1,77 @@
+// Copyright (c) The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <chainparams.h>
+#include <index/txospenderindex.h>
+#include <test/util/setup_common.h>
+#include <util/time.h>
+#include <validation.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(txospenderindex_tests)
+
+BOOST_FIXTURE_TEST_CASE(txospenderindex_initial_sync, TestChain100Setup)
+{
+    TxoSpenderIndex txospenderindex(interfaces::MakeChain(m_node), 1 << 20, true);
+    BOOST_REQUIRE(txospenderindex.Init());
+
+    // Mine blocks for coinbase maturity, so we can spend some coinbase outputs in the test.
+    for (int i = 0; i < 50; i++) {
+        std::vector<CMutableTransaction> no_txns;
+        CreateAndProcessBlock(no_txns, this->m_coinbase_txns[i]->vout[0].scriptPubKey);
+    }
+    std::vector<COutPoint> spent(10);
+    std::vector<CMutableTransaction> spender(spent.size());
+
+    for (size_t i = 0; i < spent.size(); i++) {
+        spent[i] = COutPoint(this->m_coinbase_txns[i]->GetHash(), 0);
+        spender[i].version = 1;
+        spender[i].vin.resize(1);
+        spender[i].vin[0].prevout.hash = spent[i].hash;
+        spender[i].vin[0].prevout.n = spent[i].n;
+        spender[i].vout.resize(1);
+        spender[i].vout[0].nValue = this->m_coinbase_txns[i]->GetValueOut();
+        spender[i].vout[0].scriptPubKey = this->m_coinbase_txns[i]->vout[0].scriptPubKey;
+
+        // Sign:
+        std::vector<unsigned char> vchSig;
+        const uint256 hash = SignatureHash(this->m_coinbase_txns[i]->vout[0].scriptPubKey, spender[i], 0, SIGHASH_ALL, 0, SigVersion::BASE);
+        coinbaseKey.Sign(hash, vchSig);
+        vchSig.push_back((unsigned char)SIGHASH_ALL);
+        spender[i].vin[0].scriptSig << vchSig;
+    }
+
+    CBlock block = CreateAndProcessBlock(spender, this->m_coinbase_txns[0]->vout[0].scriptPubKey);
+
+    // Transaction should not be found in the index before it is started.
+    for (const auto& outpoint : spent) {
+        BOOST_CHECK(!txospenderindex.FindSpender(outpoint).value());
+    }
+
+    // BlockUntilSyncedToCurrentChain should return false before txospenderindex is started.
+    BOOST_CHECK(!txospenderindex.BlockUntilSyncedToCurrentChain());
+
+    txospenderindex.Sync();
+    for (size_t i = 0; i < spent.size(); i++) {
+        const auto tx_spender{txospenderindex.FindSpender(spent[i])};
+        BOOST_REQUIRE(tx_spender.has_value());
+        BOOST_REQUIRE(tx_spender->has_value());
+        BOOST_CHECK_EQUAL((*tx_spender)->tx->GetHash(), spender[i].GetHash());
+        BOOST_CHECK_EQUAL((*tx_spender)->block_hash, block.GetHash());
+    }
+
+    // It is not safe to stop and destroy the index until it finishes handling
+    // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain()
+    // call above is sufficient to ensure this, but the
+    // SyncWithValidationInterfaceQueue() call below is also needed to ensure
+    // TSAN always sees the test thread waiting for the notification thread, and
+    // avoid potential false positive reports.
+    m_node.validation_signals->SyncWithValidationInterfaceQueue();
+
+    // shutdown sequence (c.f. Shutdown() in init.cpp)
+    txospenderindex.Stop();
+}
+
+BOOST_AUTO_TEST_SUITE_END()