Merge bitcoin/bitcoin#35482: fuzz: exercise the transaction-handling path in process_message(s)

87b080fe2b fuzz: reset the reused mempool in process_message(s) (Hao Xu)
d522fd3196 fuzz: prepare deterministic mempool rebuilds (Hao Xu)
b11456386b fuzz: let the test input toggle IBD in the p2p fuzz targets (Hao Xu)
2a29cee684 test: add helper to reset chainman and mempool (Hao Xu)
2a4ef42d34 fuzz: share a single FakeNodeClock in the chainman-resetting fuzz targets (Hao Xu)

Pull request description:

  ## Problem

  `process_message` and `process_messages` keep the node in IBD (`ResetIbd()`) and
  mine their coinbases with the default bare-`OP_TRUE` output script. As a result
  `net_processing` returns early at the `IsInitialBlockDownload()` check and never
  reaches the transaction-handling path; and even if it did, a tx spending a
  bare-`OP_TRUE` coinbase is rejected as `NONSTANDARD` by
  `ValidateInputsStandardness`. The reused mempool therefore always stays empty and
  that path is never exercised.

  ## Changes

  Both targets now get the same treatment:

  1. **Toggle IBD from the test input** — a `bool` decides whether to also
     `JumpOutOfIbd()`, exercising both the IBD and non-IBD paths. In
     `process_message` it is consumed last, so existing corpus entries read `false`
     and are unchanged. In `process_messages` the messages run in a loop, so the
     bool must be consumed *first* (see the corpus note below).
  2. **Use a spendable `P2WSH_OP_TRUE` coinbase** — both anyone-can-spend (an
     `OP_TRUE` witness, no signature) and a standard witness output, so a fuzz-built
     tx spending a mature coinbase can actually be accepted into the mempool.
  3. **Reset the rng before rebuilding (preparation)** — rebuilding the chainman
     (and, in the next commit, the mempool) consumes the global PRNG. Reset it with
     `MakeRandDeterministicDANGEROUS()` first so the rebuild is deterministic across
     iterations. Mirrors the `cmpctblock` harness.
  4. **Reset the reused mempool** — now that the mempool can become non-empty,
     rebuild it together with the chainman in `ResetChainmanAndMempool()` when the
     block index grew or the mempool changed. A dirty mempool is detected by its
     sequence number rather than its size, since a tx can be added and removed
     within one iteration (leaving the size unchanged).

  ## Corpus note

  ~~In `process_messages` the IBD bool is consumed before the message loop (first
  integral read), which shifts the `FuzzedDataProvider` layout. Existing
  `process_messages` corpus entries can be migrated by appending a single `0x00`
  byte at the end (read as `false`, keeping the IBD path); every other consumed
  value stays the same. This is a qa-assets change accompanying this PR.~~

    This note no longer applies because the IBD toggle is now consumed inside the
    message loop. Appending a single `0x00` byte would not reliably target that bool
    or preserve the rest of the input layout.

    The accompanying `qa-assets` update should migrate or regenerate the affected
    `process_messages` corpus entries for the current layout.

ACKs for top commit:
  Crypt-iQ:
    crACK 87b080fe2b
  maflcko:
    review ACK 87b080fe2b 🏁
  frankomosh:
    Review ACK 87b080fe2b

Tree-SHA512: e557b2ca3329767a45fe8315c63df9c3191a3a46a17c5e75ea3e4ad0c25e0e500a687fa650297a386b0a2ebb95503d069089ca5ae3d0a34caab98367aeb28683

src/test/fuzz/cmpctblock.cpp

@@ -14,7 +14,6 @@
 #include <net_processing.h>
 #include <netmessagemaker.h>
 #include <node/blockstorage.h>
-#include <node/mining_types.h>
 #include <policy/truc_policy.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
@@ -26,20 +25,17 @@
 #include <test/fuzz/fuzz.h>
 #include <test/fuzz/util.h>
 #include <test/fuzz/util/net.h>
-#include <test/util/mining.h>
 #include <test/util/net.h>
 #include <test/util/random.h>
 #include <test/util/script.h>
 #include <test/util/setup_common.h>
 #include <test/util/time.h>
-#include <test/util/txmempool.h>
 #include <test/util/validation.h>
 #include <txmempool.h>
 #include <uint256.h>
 #include <util/check.h>
 #include <util/task_runner.h>
 #include <util/time.h>
-#include <util/translation.h>
 #include <validation.h>
 #include <validationinterface.h>
 
@@ -107,33 +103,6 @@ public:
     }
 };
 
-void ResetChainmanAndMempool(TestingSetup& setup)
-{
-    SetMockTime(Params().GenesisBlock().Time());
-
-    bilingual_str error{};
-    setup.m_node.mempool.reset();
-    setup.m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(setup.m_node), error);
-    Assert(error.empty());
-
-    setup.m_node.chainman.reset();
-    setup.m_make_chainman();
-    setup.LoadVerifyActivateChainstate();
-
-    node::BlockCreateOptions options;
-    options.coinbase_output_script = P2WSH_OP_TRUE;
-
-    g_mature_coinbase.clear();
-
-    for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) {
-        COutPoint prevout{MineBlock(setup.m_node, options)};
-        if (i < COINBASE_MATURITY) {
-            LOCK(cs_main);
-            CAmount subsidy{setup.m_node.chainman->ActiveChainstate().CoinsTip().GetCoin(prevout)->out.nValue};
-            g_mature_coinbase.emplace_back(prevout, subsidy);
-        }
-    }
-}
 
 //! Used to run tasks in a std::thread to avoid DEBUG_LOCKORDER false positives.
 class ImmediateBackgroundTaskRunner : public util::TaskRunnerInterface
@@ -155,7 +124,7 @@ void initialize_cmpctblock()
     g_nBits = Params().GenesisBlock().nBits;
     // Replace validation_signals before creating chainman and mempool so they use it.
     testing_setup->m_node.validation_signals = std::make_unique<ValidationSignals>(std::make_unique<ImmediateBackgroundTaskRunner>());
-    ResetChainmanAndMempool(*g_setup);
+    g_mature_coinbase = ResetChainmanAndMempool(*g_setup);
 }
 
 FUZZ_TARGET(cmpctblock, .init = initialize_cmpctblock)
@@ -163,7 +132,7 @@ FUZZ_TARGET(cmpctblock, .init = initialize_cmpctblock)
     SeedRandomStateForTest(SeedRand::ZEROS);
     FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
 
-    FakeNodeClock clock{1610000000s};
+    GetFakeNodeClock().set(1610000000s);
     FakeSteadyClock steady_clock;
 
     auto setup = g_setup;
@@ -453,10 +422,10 @@ FUZZ_TARGET(cmpctblock, .init = initialize_cmpctblock)
             [&]() {
                 // Set mock time randomly or to tip's time.
                 if (fuzzed_data_provider.ConsumeBool()) {
-                    clock.set(ConsumeTime(fuzzed_data_provider));
+                    GetFakeNodeClock().set(ConsumeTime(fuzzed_data_provider));
                 } else {
                     const NodeSeconds tip_time = WITH_LOCK(::cs_main, return chainman.ActiveChain().Tip()->Time());
-                    clock.set(tip_time);
+                    GetFakeNodeClock().set(tip_time);
                 }
 
                 sent_net_msg = false;
@@ -509,6 +478,6 @@ FUZZ_TARGET(cmpctblock, .init = initialize_cmpctblock)
 
     if (initial_index_size != end_index_size || initial_sequence != end_sequence) {
         MakeRandDeterministicDANGEROUS(uint256::ZERO);
-        ResetChainmanAndMempool(*g_setup);
+        g_mature_coinbase = ResetChainmanAndMempool(*g_setup);
     }
 }

src/test/fuzz/p2p_handshake.cpp

@@ -41,7 +41,7 @@ FUZZ_TARGET(p2p_handshake, .init = ::initialize)
     auto& node{g_setup->m_node};
     auto& connman{static_cast<ConnmanTestMsg&>(*node.connman)};
     auto& chainman{static_cast<TestChainstateManager&>(*node.chainman)};
-    FakeNodeClock clock{1610000000s}; // any time to successfully reset ibd
+    FakeNodeClock clock{1610000000s}; // 2021-01-07, arbitrary
     FakeSteadyClock steady_clock;
     chainman.ResetIbd();
 
@@ -72,6 +72,10 @@ FUZZ_TARGET(p2p_handshake, .init = ::initialize)
             static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>()));
     }
 
+    // Toggle IBD from within the loop, so that some messages may be processed
+    // under IBD and the rest after leaving it. JumpOutOfIbd() latches, so guard
+    // it to call at most once.
+    bool jump_out_of_ibd{false};
     LIMITED_WHILE (fuzzed_data_provider.ConsumeBool(), 100) {
         CNode& connection = *PickValue(fuzzed_data_provider, peers);
         if (connection.fDisconnect || connection.fSuccessfullyConnected) {
@@ -80,6 +84,9 @@ FUZZ_TARGET(p2p_handshake, .init = ::initialize)
             continue;
         }
 
+        if (!jump_out_of_ibd) jump_out_of_ibd = fuzzed_data_provider.ConsumeBool();
+        if (jump_out_of_ibd && chainman.IsInitialBlockDownload()) chainman.JumpOutOfIbd();
+
         clock += std::chrono::seconds{
                     fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(
                         -std::chrono::seconds{10min}.count(), // Allow mocktime to go backwards slightly

src/test/fuzz/process_message.cpp

@@ -4,11 +4,9 @@
 
 #include <addrman.h>
 #include <banman.h>
-#include <consensus/consensus.h>
 #include <kernel/chainparams.h>
 #include <net.h>
 #include <net_processing.h>
-#include <node/mining_types.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
 #include <protocol.h>
@@ -17,12 +15,12 @@
 #include <test/fuzz/fuzz.h>
 #include <test/fuzz/util.h>
 #include <test/fuzz/util/net.h>
-#include <test/util/mining.h>
 #include <test/util/net.h>
 #include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <test/util/time.h>
 #include <test/util/validation.h>
+#include <uint256.h>
 #include <util/check.h>
 #include <util/time.h>
 #include <validation.h>
@@ -44,19 +42,10 @@ namespace {
 TestingSetup* g_setup;
 std::string_view LIMIT_TO_MESSAGE_TYPE{};
 
-void ResetChainman(TestingSetup& setup)
-{
-    SetMockTime(setup.m_node.chainman->GetParams().GenesisBlock().Time());
-    setup.m_node.chainman.reset();
-    setup.m_make_chainman();
-    setup.LoadVerifyActivateChainstate();
-    for (int i = 0; i < 2 * COINBASE_MATURITY; i++) {
-        node::BlockCreateOptions options;
-        MineBlock(setup.m_node, options);
-    }
-}
 } // namespace
 
+extern void MakeRandDeterministicDANGEROUS(const uint256& seed) noexcept;
+
 void initialize_process_message()
 {
     if (const auto val{std::getenv("LIMIT_TO_MESSAGE_TYPE")}) {
@@ -70,7 +59,7 @@ void initialize_process_message()
             {}),
     };
     g_setup = testing_setup.get();
-    ResetChainman(*g_setup);
+    ResetChainmanAndMempool(*g_setup);
 }
 
 FUZZ_TARGET(process_message, .init = initialize_process_message)
@@ -83,7 +72,8 @@ FUZZ_TARGET(process_message, .init = initialize_process_message)
     connman.Reset();
     auto& chainman{static_cast<TestChainstateManager&>(*node.chainman)};
     const auto block_index_size{WITH_LOCK(chainman.GetMutex(), return chainman.BlockIndex().size())};
-    FakeNodeClock clock{1610000000s}; // any time to successfully reset ibd
+    const auto initial_sequence{WITH_LOCK(node.mempool->cs, return node.mempool->GetSequence())};
+    GetFakeNodeClock().set(1610000000s); // 2021-01-07, arbitrary
     FakeSteadyClock steady_clock;
     chainman.ResetIbd();
     chainman.DisableNextWrite();
@@ -117,7 +107,7 @@ FUZZ_TARGET(process_message, .init = initialize_process_message)
     connman.AddTestNode(p2p_node);
     FillNode(fuzzed_data_provider, connman, p2p_node);
 
-    clock.set(ConsumeTime(fuzzed_data_provider));
+    GetFakeNodeClock().set(ConsumeTime(fuzzed_data_provider));
 
     CSerializedNetMsg net_msg;
     net_msg.m_type = random_message_type;
@@ -126,6 +116,10 @@ FUZZ_TARGET(process_message, .init = initialize_process_message)
     connman.FlushSendBuffer(p2p_node);
     (void)connman.ReceiveMsgFrom(p2p_node, std::move(net_msg));
 
+    if (fuzzed_data_provider.ConsumeBool()) {
+        chainman.JumpOutOfIbd();
+    }
+
     bool more_work{true};
     while (more_work) {
         p2p_node.fPauseSend = false;
@@ -138,8 +132,10 @@ FUZZ_TARGET(process_message, .init = initialize_process_message)
     node.validation_signals->SyncWithValidationInterfaceQueue();
     node.validation_signals->UnregisterValidationInterface(node.peerman.get());
     node.connman->StopNodes();
-    if (block_index_size != WITH_LOCK(chainman.GetMutex(), return chainman.BlockIndex().size())) {
-        // Reuse the global chainman, but reset it when it is dirty
-        ResetChainman(*g_setup);
+    const auto end_sequence{WITH_LOCK(node.mempool->cs, return node.mempool->GetSequence())};
+    if (block_index_size != WITH_LOCK(chainman.GetMutex(), return chainman.BlockIndex().size()) || initial_sequence != end_sequence) {
+        // Reuse the global chainman and mempool, but reset them when dirty.
+        MakeRandDeterministicDANGEROUS(uint256::ZERO);
+        ResetChainmanAndMempool(*g_setup);
     }
 }

src/test/fuzz/process_messages.cpp

@@ -4,11 +4,9 @@
 
 #include <addrman.h>
 #include <banman.h>
-#include <consensus/consensus.h>
 #include <kernel/chainparams.h>
 #include <net.h>
 #include <net_processing.h>
-#include <node/mining_types.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
 #include <protocol.h>
@@ -17,12 +15,13 @@
 #include <test/fuzz/fuzz.h>
 #include <test/fuzz/util.h>
 #include <test/fuzz/util/net.h>
-#include <test/util/mining.h>
 #include <test/util/net.h>
 #include <test/util/random.h>
 #include <test/util/setup_common.h>
 #include <test/util/time.h>
 #include <test/util/validation.h>
+#include <uint256.h>
+#include <util/check.h>
 #include <util/time.h>
 #include <validation.h>
 #include <validationinterface.h>
@@ -38,19 +37,10 @@
 namespace {
 TestingSetup* g_setup;
 
-void ResetChainman(TestingSetup& setup)
-{
-    SetMockTime(setup.m_node.chainman->GetParams().GenesisBlock().Time());
-    setup.m_node.chainman.reset();
-    setup.m_make_chainman();
-    setup.LoadVerifyActivateChainstate();
-    node::BlockCreateOptions options;
-    for (int i = 0; i < 2 * COINBASE_MATURITY; i++) {
-        MineBlock(setup.m_node, options);
-    }
-}
 } // namespace
 
+extern void MakeRandDeterministicDANGEROUS(const uint256& seed) noexcept;
+
 void initialize_process_messages()
 {
     static const auto testing_setup{
@@ -59,7 +49,7 @@ void initialize_process_messages()
             {}),
     };
     g_setup = testing_setup.get();
-    ResetChainman(*g_setup);
+    ResetChainmanAndMempool(*g_setup);
 }
 
 FUZZ_TARGET(process_messages, .init = initialize_process_messages)
@@ -72,7 +62,8 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages)
     connman.Reset();
     auto& chainman{static_cast<TestChainstateManager&>(*node.chainman)};
     const auto block_index_size{WITH_LOCK(chainman.GetMutex(), return chainman.BlockIndex().size())};
-    FakeNodeClock clock{1610000000s}; // any time to successfully reset ibd
+    const auto initial_sequence{WITH_LOCK(node.mempool->cs, return node.mempool->GetSequence())};
+    GetFakeNodeClock().set(1610000000s); // 2021-01-07, arbitrary
     FakeSteadyClock steady_clock;
     chainman.ResetIbd();
     chainman.DisableNextWrite();
@@ -107,10 +98,16 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages)
         connman.AddTestNode(p2p_node);
     }
 
+    // Toggle IBD from within the loop, so that some messages may be processed
+    // under IBD and the rest after leaving it. JumpOutOfIbd() latches, so guard
+    // it to call at most once.
+    bool jump_out_of_ibd{false};
     LIMITED_WHILE (fuzzed_data_provider.ConsumeBool(), 30) {
+        if (!jump_out_of_ibd) jump_out_of_ibd = fuzzed_data_provider.ConsumeBool();
+        if (jump_out_of_ibd && chainman.IsInitialBlockDownload()) chainman.JumpOutOfIbd();
         const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::MESSAGE_TYPE_SIZE).c_str()};
 
-        clock.set(ConsumeTime(fuzzed_data_provider));
+        GetFakeNodeClock().set(ConsumeTime(fuzzed_data_provider));
 
         CSerializedNetMsg net_msg;
         net_msg.m_type = random_message_type;
@@ -135,8 +132,10 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages)
     node.validation_signals->SyncWithValidationInterfaceQueue();
     node.validation_signals->UnregisterValidationInterface(node.peerman.get());
     node.connman->StopNodes();
-    if (block_index_size != WITH_LOCK(chainman.GetMutex(), return chainman.BlockIndex().size())) {
-        // Reuse the global chainman, but reset it when it is dirty
-        ResetChainman(*g_setup);
+    const auto end_sequence{WITH_LOCK(node.mempool->cs, return node.mempool->GetSequence())};
+    if (block_index_size != WITH_LOCK(chainman.GetMutex(), return chainman.BlockIndex().size()) || initial_sequence != end_sequence) {
+        // Reuse the global chainman and mempool, but reset them when dirty.
+        MakeRandDeterministicDANGEROUS(uint256::ZERO);
+        ResetChainmanAndMempool(*g_setup);
     }
 }

src/test/fuzz/utxo_snapshot.cpp

@@ -73,7 +73,7 @@ void initialize_chain()
     const auto params{CreateChainParams(ArgsManager{}, ChainType::REGTEST)};
     static const auto chain{CreateBlockChain(2 * COINBASE_MATURITY, *params)};
     g_chain = &chain;
-    SetMockTime(chain.back()->Time());
+    GetFakeNodeClock().set(chain.back()->Time());
 
     // Make sure we can generate a valid snapshot.
     sanity_check_snapshot();
@@ -104,7 +104,7 @@ void utxo_snapshot_fuzz(FuzzBufferType buffer)
 {
     SeedRandomStateForTest(SeedRand::ZEROS);
     FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
-    FakeNodeClock clock{ConsumeTime(fuzzed_data_provider, /*min=*/1296688602)}; // regtest genesis block timestamp
+    GetFakeNodeClock().set(ConsumeTime(fuzzed_data_provider, /*min=*/1296688602)); // regtest genesis block timestamp
     auto& setup{*g_setup};
     bool dirty_chainman{false}; // Reuse the global chainman, but reset it when it is dirty
     auto& chainman{*setup.m_node.chainman};

src/test/util/time.h

@@ -76,4 +76,10 @@ public:
     void operator-=(std::chrono::seconds d) { set(m_t -= d); }
 };
 
+inline FakeNodeClock& GetFakeNodeClock()
+{
+    static FakeNodeClock g_fake_node_clock{0s};
+    return g_fake_node_clock;
+}
+
 #endif // BITCOIN_TEST_UTIL_TIME_H

src/test/util/validation.cpp

@@ -4,12 +4,25 @@
 
 #include <test/util/validation.h>
 
+#include <coins.h>
+#include <consensus/consensus.h>
 #include <node/blockstorage.h>
+#include <node/mining_types.h>
+#include <test/util/mining.h>
+#include <test/util/script.h>
+#include <test/util/setup_common.h>
+#include <test/util/time.h>
+#include <test/util/txmempool.h>
+#include <txmempool.h>
 #include <util/check.h>
 #include <util/time.h>
 #include <validation.h>
 #include <validationinterface.h>
 
+#include <memory>
+#include <utility>
+#include <vector>
+
 using kernel::ChainstateRole;
 
 void TestBlockManager::CleanupForFuzzing()
@@ -91,3 +104,31 @@ void TestChainstateManager::ResetBestInvalid()
 {
     m_best_invalid = nullptr;
 }
+
+std::vector<std::pair<COutPoint, CAmount>> ResetChainmanAndMempool(TestingSetup& setup)
+{
+    GetFakeNodeClock().set(setup.m_node.chainman->GetParams().GenesisBlock().Time());
+
+    bilingual_str error{};
+    setup.m_node.mempool.reset();
+    setup.m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(setup.m_node), error);
+    Assert(error.empty());
+
+    setup.m_node.chainman.reset();
+    setup.m_make_chainman();
+    setup.LoadVerifyActivateChainstate();
+
+    node::BlockCreateOptions options;
+    options.coinbase_output_script = P2WSH_OP_TRUE;
+
+    std::vector<std::pair<COutPoint, CAmount>> mature_coinbase;
+    for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) {
+        COutPoint prevout{MineBlock(setup.m_node, options)};
+        if (i < COINBASE_MATURITY) {
+            LOCK(cs_main);
+            CAmount subsidy{setup.m_node.chainman->ActiveChainstate().CoinsTip().GetCoin(prevout)->out.nValue};
+            mature_coinbase.emplace_back(prevout, subsidy);
+        }
+    }
+    return mature_coinbase;
+}

src/test/util/validation.h

@@ -5,12 +5,18 @@
 #ifndef BITCOIN_TEST_UTIL_VALIDATION_H
 #define BITCOIN_TEST_UTIL_VALIDATION_H
 
+#include <consensus/amount.h>
+#include <primitives/transaction.h>
 #include <validation.h>
 
+#include <utility>
+#include <vector>
+
 namespace node {
 class BlockManager;
 }
 class CValidationInterface;
+struct TestingSetup;
 
 struct TestBlockManager : public node::BlockManager {
     /** Test-only method to clear internal state for fuzzing */
@@ -41,4 +47,6 @@ public:
         const CBlockIndex* pindex);
 };
 
+std::vector<std::pair<COutPoint, CAmount>> ResetChainmanAndMempool(TestingSetup& setup);
+
 #endif // BITCOIN_TEST_UTIL_VALIDATION_H