Merge bitcoin/bitcoin#27607: index: make startup more efficient

ca91c244ef index: verify blocks data existence only once (furszy)
fcbdaeef4d init: don't start indexes sync thread prematurely (furszy)
2ec89f1970 refactor: simplify pruning violation check (furszy)
c82ef91eae make GetFirstStoredBlock assert that 'start_block' always has data (furszy)
430e7027a1 refactor: index, decouple 'Init' from 'Start' (furszy)
225e213110 refactor: init indexes, decouple 'Start()' from the creation step (furszy)
2ebc7e68cc doc: describe 'init load' thread actions (Martin Zumsande)
04575106b2 scripted-diff: rename 'loadblk' thread name to 'initload' (furszy)
ed4462cc78 init: start indexes sync earlier (furszy)

Pull request description:

  Simplifies index startup code, eliminating the `g_indexes_ready_to_sync` variable,
  deduplicating code and moving the prune violation check out of the `BaseIndex` class.

  Also makes startup more efficient by running the prune violation check once for all indexes
  instead of once for each index, and by delaying the prune violation check and moving it off
  of the main thread so the node can start up faster and perform the block data availability
  verification even when the '-reindex" or the "-reindex-chainstate" flags are enabled (which
  hasn't  being possible so far).

ACKs for top commit:
  ryanofsky:
    Code review ACK ca91c244ef. Just rebase and suggested changes since last review (Start return check, and code simplification)
  TheCharlatan:
    re-ACK ca91c244ef

Tree-SHA512: e9c98ce89aeb29e8d0f505f17b34aa54fe44efefbf017f4746e3b446ab4de25ade4f707254a0bbe4b99b69731b04a4067ce529eb7aa834ced196784b694cf7ce

src/test/blockfilter_index_tests.cpp

@@ -113,6 +113,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex,
 BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup)
 {
     BlockFilterIndex filter_index(interfaces::MakeChain(m_node), BlockFilterType::BASIC, 1 << 20, true);
+    BOOST_REQUIRE(filter_index.Init());
 
     uint256 last_header;
 
@@ -139,7 +140,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup)
     // BlockUntilSyncedToCurrentChain should return false before index is started.
     BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain());
 
-    BOOST_REQUIRE(filter_index.Start());
+    BOOST_REQUIRE(filter_index.StartBackgroundSync());
 
     // Allow filter index to catch up with the block index.
     IndexWaitSynced(filter_index);

src/test/blockmanager_tests.cpp

@@ -90,4 +90,43 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain
     BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(new_pos, true)).IsNull());
 }
 
+BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup)
+{
+    // The goal of the function is to return the first not pruned block in the range [upper_block, lower_block].
+    LOCK(::cs_main);
+    auto& chainman = m_node.chainman;
+    auto& blockman = chainman->m_blockman;
+    const CBlockIndex& tip = *chainman->ActiveTip();
+
+    // Function to prune all blocks from 'last_pruned_block' down to the genesis block
+    const auto& func_prune_blocks = [&](CBlockIndex* last_pruned_block)
+    {
+        LOCK(::cs_main);
+        CBlockIndex* it = last_pruned_block;
+        while (it != nullptr && it->nStatus & BLOCK_HAVE_DATA) {
+            it->nStatus &= ~BLOCK_HAVE_DATA;
+            it = it->pprev;
+        }
+    };
+
+    // 1) Return genesis block when all blocks are available
+    BOOST_CHECK_EQUAL(blockman.GetFirstStoredBlock(tip), chainman->ActiveChain()[0]);
+    BOOST_CHECK(blockman.CheckBlockDataAvailability(tip, *chainman->ActiveChain()[0]));
+
+    // 2) Check lower_block when all blocks are available
+    CBlockIndex* lower_block = chainman->ActiveChain()[tip.nHeight / 2];
+    BOOST_CHECK(blockman.CheckBlockDataAvailability(tip, *lower_block));
+
+    // Prune half of the blocks
+    int height_to_prune = tip.nHeight / 2;
+    CBlockIndex* first_available_block = chainman->ActiveChain()[height_to_prune + 1];
+    CBlockIndex* last_pruned_block = first_available_block->pprev;
+    func_prune_blocks(last_pruned_block);
+
+    // 3) The last block not pruned is in-between upper-block and the genesis block
+    BOOST_CHECK_EQUAL(blockman.GetFirstStoredBlock(tip), first_available_block);
+    BOOST_CHECK(blockman.CheckBlockDataAvailability(tip, *first_available_block));
+    BOOST_CHECK(!blockman.CheckBlockDataAvailability(tip, *last_pruned_block));
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/test/coinstatsindex_tests.cpp

@@ -18,6 +18,7 @@ BOOST_AUTO_TEST_SUITE(coinstatsindex_tests)
 BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup)
 {
     CoinStatsIndex coin_stats_index{interfaces::MakeChain(m_node), 1 << 20, true};
+    BOOST_REQUIRE(coin_stats_index.Init());
 
     const CBlockIndex* block_index;
     {
@@ -32,7 +33,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup)
     // is started.
     BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain());
 
-    BOOST_REQUIRE(coin_stats_index.Start());
+    BOOST_REQUIRE(coin_stats_index.StartBackgroundSync());
 
     IndexWaitSynced(coin_stats_index);
 
@@ -83,7 +84,8 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup)
     const CChainParams& params = Params();
     {
         CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20};
-        BOOST_REQUIRE(index.Start());
+        BOOST_REQUIRE(index.Init());
+        BOOST_REQUIRE(index.StartBackgroundSync());
         IndexWaitSynced(index);
         std::shared_ptr<const CBlock> new_block;
         CBlockIndex* new_block_index = nullptr;
@@ -109,8 +111,9 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup)
 
     {
         CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20};
+        BOOST_REQUIRE(index.Init());
         // Make sure the index can be loaded.
-        BOOST_REQUIRE(index.Start());
+        BOOST_REQUIRE(index.StartBackgroundSync());
         index.Stop();
     }
 }

src/test/txindex_tests.cpp

@@ -17,6 +17,7 @@ BOOST_AUTO_TEST_SUITE(txindex_tests)
 BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup)
 {
     TxIndex txindex(interfaces::MakeChain(m_node), 1 << 20, true);
+    BOOST_REQUIRE(txindex.Init());
 
     CTransactionRef tx_disk;
     uint256 block_hash;
@@ -29,7 +30,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup)
     // BlockUntilSyncedToCurrentChain should return false before txindex is started.
     BOOST_CHECK(!txindex.BlockUntilSyncedToCurrentChain());
 
-    BOOST_REQUIRE(txindex.Start());
+    BOOST_REQUIRE(txindex.StartBackgroundSync());
 
     // Allow tx index to catch up with the block index.
     IndexWaitSynced(txindex);

src/test/util/setup_common.cpp

@@ -157,7 +157,6 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto
         noui_connect();
         noui_connected = true;
     }
-    node::g_indexes_ready_to_sync = true;
 }
 
 BasicTestingSetup::~BasicTestingSetup()