Merge bitcoin/bitcoin#29770: index: Check all necessary block data is available before starting to sync

fd06157d14 test: Add coverage for restarted node without any block sync (Fabian Jahr)
3d7ab7ecb7 rpc, test: Address feedback from #29668 (Fabian Jahr)
312919c9dd test: Indices can not start based on block data without undo data (Fabian Jahr)
a9a3b29dd6 index: Check availability of undo data for indices (Fabian Jahr)
881ab4fc82 support multiple block status checks in CheckBlockDataAvailability (furszy)

Pull request description:

  Currently, we check that `BLOCK_HAVE_DATA` is available for all blocks an index needs to sync during startup. However, for `coinstatsindex` and `blockfilterindex` we also need the undo data for these blocks. If that data is missing in the blocks, we are currently still starting to sync each of these indices and then crash later when we encounter the missing data.

  This PR adds explicit knowledge of which block data is needed for each index and then checks its availability during startup before initializing the sync process on them.

  This also addresses a few open comments from #29668 in the last commit.

ACKs for top commit:
  achow101:
    ACK fd06157d14
  furszy:
    utACK fd06157d14
  sedited:
    Re-ACK fd06157d14

Tree-SHA512: e2ed81c93372b02daa8ddf2819df4164f96d92de05b1d48855410ecac78d5fcd9612d7f0e63a9d57d7e75a0b46e1bea278e43ea87f2693af0220d1f9c600e416

test/functional/feature_index_prune.py

@@ -5,13 +5,28 @@
 """Test indices in conjunction with prune."""
 import concurrent.futures
 import os
+from test_framework.authproxy import JSONRPCException
 from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_node import TestNode
 from test_framework.util import (
     assert_equal,
     assert_greater_than,
     assert_raises_rpc_error,
 )
 
+from typing import List, Any
+
+def send_batch_request(node: TestNode, method: str, params: List[Any]) -> List[Any]:
+    """Send batch request and parse all results"""
+    data = [{"method": method, "params": p} for p in params]
+    response = node.batch(data)
+    result = []
+    for item in response:
+        assert item["error"] is None, item["error"]
+        result.append(item["result"])
+
+    return result
+
 
 class FeatureIndexPruneTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -57,6 +72,13 @@ class FeatureIndexPruneTest(BitcoinTestFramework):
         for i in range(3):
             self.restart_node(i, extra_args=["-fastprune", "-prune=1"])
 
+    def check_for_block(self, node, hash):
+        try:
+            self.nodes[node].getblock(hash)
+            return True
+        except JSONRPCException:
+            return False
+
     def run_test(self):
         filter_nodes = [self.nodes[0], self.nodes[2]]
         stats_nodes = [self.nodes[1], self.nodes[2]]
@@ -130,12 +152,38 @@ class FeatureIndexPruneTest(BitcoinTestFramework):
             self.stop_node(i)
 
         self.log.info("make sure we get an init error when starting the nodes again with the indices")
-        filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
-        stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
+        filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)"
+        stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data (including undo data). Please disable the index or reindex (which will download the whole blockchain again)"
         end_msg = f"{os.linesep}Error: A fatal internal error occurred, see debug.log for details: Failed to start indexes, shutting down…"
         for i, msg in enumerate([filter_msg, stats_msg, filter_msg]):
             self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)
 
+        self.log.info("fetching the missing blocks with getblockfrompeer doesn't work for block filter index and coinstatsindex")
+        # Only checking the first two nodes since this test takes a long time
+        # and the third node is kind of redundant in this context
+        for i, msg in enumerate([filter_msg, stats_msg]):
+            self.restart_node(i, extra_args=["-prune=1", "-fastprune"])
+            node = self.nodes[i]
+            prune_height = node.getblockchaininfo()["pruneheight"]
+            self.connect_nodes(i, 3)
+            peers = node.getpeerinfo()
+            assert_equal(len(peers), 1)
+            peer_id = peers[0]["id"]
+
+            # 1500 is the height to where the indices were able to sync previously
+            hashes = send_batch_request(node, "getblockhash", [[a] for a in range(1500, prune_height)])
+            send_batch_request(node, "getblockfrompeer", [[bh, peer_id] for bh in hashes])
+            # Ensure all necessary blocks have been fetched before proceeding
+            for bh in hashes:
+                self.wait_until(lambda: self.check_for_block(i, bh), timeout=10)
+
+            # Upon restart we expect the same errors as previously although all
+            # necessary blocks have been fetched. Both indices need the undo
+            # data of the blocks to be available as well and getblockfrompeer
+            # can not provide that.
+            self.stop_node(i)
+            node.assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg+end_msg)
+
         self.log.info("make sure the nodes start again with the indices and an additional -reindex arg")
         for i in range(3):
             restart_args = self.extra_args[i] + ["-reindex"]

test/functional/feature_init.py

@@ -26,8 +26,8 @@ class InitTest(BitcoinTestFramework):
     """
 
     def set_test_params(self):
-        self.setup_clean_chain = False
-        self.num_nodes = 1
+        self.setup_clean_chain = True
+        self.num_nodes = 2
         self.uses_wallet = None
 
     def check_clean_start(self, node, extra_args):
@@ -42,8 +42,10 @@ class InitTest(BitcoinTestFramework):
         """
         - test terminating initialization after seeing a certain log line.
         """
-        self.stop_node(0)
+        self.start_node(0)
         node = self.nodes[0]
+        self.generate(node, 200, sync_fun=self.no_op)
+        self.stop_node(0)
 
         def sigterm_node():
             if platform.system() == 'Windows':
@@ -307,11 +309,21 @@ class InitTest(BitcoinTestFramework):
             assert_equal(result["height"], current_height)
             node.wait_until_stopped()
 
+    def init_empty_test(self):
+        self.log.info("Test that stopping and restarting a node that has done nothing is not causing a failure")
+        options = [
+            [],
+            ["-txindex=1", "-blockfilterindex=1", "-coinstatsindex=1"],
+        ]
+        for option in options:
+            self.restart_node(1, option)
+
     def run_test(self):
         self.init_pid_test()
         self.init_stress_test_interrupt()
         self.init_stress_test_removals()
         self.break_wait_test()
+        self.init_empty_test()
 
 
 if __name__ == '__main__':