diff --git a/src/init.cpp b/src/init.cpp index 47cffdd335e..96fec92133d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1555,11 +1555,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - auto result{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}; - if (!result) { - return InitError(util::ErrorString(result)); - } - g_txindex = std::make_unique(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); node.indexes.emplace_back(g_txindex.get()); } diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 01b4c36a8f3..70f11be5861 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -15,15 +16,125 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include +namespace kernel { +static constexpr uint8_t DB_BLOCK_FILES{'f'}; +static constexpr uint8_t DB_BLOCK_INDEX{'b'}; +static constexpr uint8_t DB_FLAG{'F'}; +static constexpr uint8_t DB_REINDEX_FLAG{'R'}; +static constexpr uint8_t DB_LAST_BLOCK{'l'}; +// Keys used in previous version that might still be found in the DB: +// BlockTreeDB::DB_TXINDEX_BLOCK{'T'}; +// BlockTreeDB::DB_TXINDEX{'t'} +// BlockTreeDB::ReadFlag("txindex") + +bool BlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo& info) +{ + return Read(std::make_pair(DB_BLOCK_FILES, nFile), info); +} + +bool BlockTreeDB::WriteReindexing(bool fReindexing) +{ + if (fReindexing) { + return Write(DB_REINDEX_FLAG, uint8_t{'1'}); + } else { + return Erase(DB_REINDEX_FLAG); + } +} + +void BlockTreeDB::ReadReindexing(bool& fReindexing) +{ + fReindexing = Exists(DB_REINDEX_FLAG); +} + +bool BlockTreeDB::ReadLastBlockFile(int& nFile) +{ + return Read(DB_LAST_BLOCK, nFile); +} + +bool BlockTreeDB::WriteBatchSync(const std::vector>& fileInfo, int nLastFile, const std::vector& blockinfo) +{ + CDBBatch batch(*this); + for (const auto& [file, info] : fileInfo) { + batch.Write(std::make_pair(DB_BLOCK_FILES, file), *info); + } + batch.Write(DB_LAST_BLOCK, nLastFile); + for (const CBlockIndex* bi : blockinfo) { + batch.Write(std::make_pair(DB_BLOCK_INDEX, bi->GetBlockHash()), CDiskBlockIndex{bi}); + } + return WriteBatch(batch, true); +} + +bool BlockTreeDB::WriteFlag(const std::string& name, bool fValue) +{ + return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'}); +} + +bool BlockTreeDB::ReadFlag(const std::string& name, bool& fValue) +{ + uint8_t ch; + if (!Read(std::make_pair(DB_FLAG, name), ch)) { + return false; + } + fValue = ch == uint8_t{'1'}; + return true; +} + +bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex, const util::SignalInterrupt& interrupt) +{ + AssertLockHeld(::cs_main); + std::unique_ptr pcursor(NewIterator()); + pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256())); + + // Load m_block_index + while (pcursor->Valid()) { + if (interrupt) return false; + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { + CDiskBlockIndex diskindex; + if (pcursor->GetValue(diskindex)) { + // Construct block index object + CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash()); + pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); + pindexNew->nHeight = diskindex.nHeight; + pindexNew->nFile = diskindex.nFile; + pindexNew->nDataPos = diskindex.nDataPos; + pindexNew->nUndoPos = diskindex.nUndoPos; + pindexNew->nVersion = diskindex.nVersion; + pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; + pindexNew->nTime = diskindex.nTime; + pindexNew->nBits = diskindex.nBits; + pindexNew->nNonce = diskindex.nNonce; + pindexNew->nStatus = diskindex.nStatus; + pindexNew->nTx = diskindex.nTx; + + if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) { + return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString()); + } + + pcursor->Next(); + } else { + return error("%s: failed to read value", __func__); + } + } else { + break; + } + } + + return true; +} +} // namespace kernel + namespace node { std::atomic_bool fReindex(false); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 0180124a798..c79fd2c6a14 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -7,17 +7,25 @@ #include #include +#include #include #include #include #include #include -#include #include +#include #include #include +#include +#include +#include +#include +#include +#include #include +#include #include class BlockValidationState; @@ -36,7 +44,26 @@ namespace util { class SignalInterrupt; } // namespace util +namespace kernel { +/** Access to the block database (blocks/index/) */ +class BlockTreeDB : public CDBWrapper +{ +public: + using CDBWrapper::CDBWrapper; + bool WriteBatchSync(const std::vector>& fileInfo, int nLastFile, const std::vector& blockinfo); + bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info); + bool ReadLastBlockFile(int& nFile); + bool WriteReindexing(bool fReindexing); + void ReadReindexing(bool& fReindexing); + bool WriteFlag(const std::string& name, bool fValue); + bool ReadFlag(const std::string& name, bool& fValue); + bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex, const util::SignalInterrupt& interrupt) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); +}; +} // namespace kernel + namespace node { +using kernel::BlockTreeDB; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -185,7 +212,7 @@ public: */ std::multimap m_blocks_unlinked; - std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); + std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 0828f648567..ae1457a87ea 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -37,10 +37,10 @@ static ChainstateLoadResult CompleteChainstateInitialization( const ChainstateLoadOptions& options) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { auto& pblocktree{chainman.m_blockman.m_block_tree_db}; - // new CBlockTreeDB tries to delete the existing file, which + // new BlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); - pblocktree = std::make_unique(DBParams{ + pblocktree = std::make_unique(DBParams{ .path = chainman.m_options.datadir / "blocks" / "index", .cache_bytes = static_cast(cache_sizes.block_tree_db), .memory_only = options.block_tree_db_in_memory, diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index ecae743d140..331199709e9 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -64,6 +64,7 @@ #include #include +using kernel::BlockTreeDB; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; using node::BlockAssembler; @@ -182,7 +183,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto .notifications = chainman_opts.notifications, }; m_node.chainman = std::make_unique(m_node.kernel->interrupt, chainman_opts, blockman_opts); - m_node.chainman->m_blockman.m_block_tree_db = std::make_unique(DBParams{ + m_node.chainman->m_blockman.m_block_tree_db = std::make_unique(DBParams{ .path = m_args.GetDataDirNet() / "blocks" / "index", .cache_bytes = static_cast(m_cache_sizes.block_tree_db), .memory_only = true}); diff --git a/src/txdb.cpp b/src/txdb.cpp index 538c4d8f076..e4a4b3bf72a 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,49 +5,25 @@ #include -#include +#include +#include #include -#include +#include #include +#include #include -#include -#include #include -#include +#include +#include +#include +#include static constexpr uint8_t DB_COIN{'C'}; -static constexpr uint8_t DB_BLOCK_FILES{'f'}; -static constexpr uint8_t DB_BLOCK_INDEX{'b'}; - static constexpr uint8_t DB_BEST_BLOCK{'B'}; static constexpr uint8_t DB_HEAD_BLOCKS{'H'}; -static constexpr uint8_t DB_FLAG{'F'}; -static constexpr uint8_t DB_REINDEX_FLAG{'R'}; -static constexpr uint8_t DB_LAST_BLOCK{'l'}; - // Keys used in previous version that might still be found in the DB: static constexpr uint8_t DB_COINS{'c'}; -static constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; -// uint8_t DB_TXINDEX{'t'} - -util::Result CheckLegacyTxindex(CBlockTreeDB& block_tree_db) -{ - CBlockLocator ignored{}; - if (block_tree_db.Read(DB_TXINDEX_BLOCK, ignored)) { - return util::Error{_("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex.")}; - } - bool txindex_legacy_flag{false}; - block_tree_db.ReadFlag("txindex", txindex_legacy_flag); - if (txindex_legacy_flag) { - // Disable legacy txindex and warn once about occupied disk space - if (!block_tree_db.WriteFlag("txindex", false)) { - return util::Error{Untranslated("Failed to write block index db flag 'txindex'='0'")}; - } - return util::Error{_("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.")}; - } - return {}; -} bool CCoinsViewDB::NeedsUpgrade() { @@ -178,25 +154,6 @@ size_t CCoinsViewDB::EstimateSize() const return m_db->EstimateSize(DB_COIN, uint8_t(DB_COIN + 1)); } -bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { - return Read(std::make_pair(DB_BLOCK_FILES, nFile), info); -} - -bool CBlockTreeDB::WriteReindexing(bool fReindexing) { - if (fReindexing) - return Write(DB_REINDEX_FLAG, uint8_t{'1'}); - else - return Erase(DB_REINDEX_FLAG); -} - -void CBlockTreeDB::ReadReindexing(bool &fReindexing) { - fReindexing = Exists(DB_REINDEX_FLAG); -} - -bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { - return Read(DB_LAST_BLOCK, nFile); -} - /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ class CCoinsViewDBCursor: public CCoinsViewCursor { @@ -269,71 +226,3 @@ void CCoinsViewDBCursor::Next() keyTmp.first = entry.key; } } - -bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CDBBatch batch(*this); - for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { - batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second); - } - batch.Write(DB_LAST_BLOCK, nLastFile); - for (std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { - batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); - } - return WriteBatch(batch, true); -} - -bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { - return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'}); -} - -bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { - uint8_t ch; - if (!Read(std::make_pair(DB_FLAG, name), ch)) - return false; - fValue = ch == uint8_t{'1'}; - return true; -} - -bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex, const util::SignalInterrupt& interrupt) -{ - AssertLockHeld(::cs_main); - std::unique_ptr pcursor(NewIterator()); - pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256())); - - // Load m_block_index - while (pcursor->Valid()) { - if (interrupt) return false; - std::pair key; - if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { - CDiskBlockIndex diskindex; - if (pcursor->GetValue(diskindex)) { - // Construct block index object - CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash()); - pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); - pindexNew->nHeight = diskindex.nHeight; - pindexNew->nFile = diskindex.nFile; - pindexNew->nDataPos = diskindex.nDataPos; - pindexNew->nUndoPos = diskindex.nUndoPos; - pindexNew->nVersion = diskindex.nVersion; - pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; - pindexNew->nTime = diskindex.nTime; - pindexNew->nBits = diskindex.nBits; - pindexNew->nNonce = diskindex.nNonce; - pindexNew->nStatus = diskindex.nStatus; - pindexNew->nTx = diskindex.nTx; - - if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) { - return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString()); - } - - pcursor->Next(); - } else { - return error("%s: failed to read value", __func__); - } - } else { - break; - } - } - - return true; -} diff --git a/src/txdb.h b/src/txdb.h index 6405437be93..c9af0a091ec 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -11,27 +11,15 @@ #include #include #include -#include #include #include -#include #include #include -#include -#include #include -class CBlockFileInfo; -class CBlockIndex; class COutPoint; class uint256; -namespace Consensus { -struct Params; -}; -namespace util { -class SignalInterrupt; -} // namespace util //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; @@ -89,22 +77,4 @@ public: std::optional StoragePath() { return m_db->StoragePath(); } }; -/** Access to the block database (blocks/index/) */ -class CBlockTreeDB : public CDBWrapper -{ -public: - using CDBWrapper::CDBWrapper; - bool WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo); - bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info); - bool ReadLastBlockFile(int &nFile); - bool WriteReindexing(bool fReindexing); - void ReadReindexing(bool &fReindexing); - bool WriteFlag(const std::string &name, bool fValue); - bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex, const util::SignalInterrupt& interrupt) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -}; - -[[nodiscard]] util::Result CheckLegacyTxindex(CBlockTreeDB& block_tree_db); - #endif // BITCOIN_TXDB_H diff --git a/src/validation.h b/src/validation.h index ba427afc64f..1ff9aaa7a34 100644 --- a/src/validation.h +++ b/src/validation.h @@ -47,7 +47,6 @@ #include class Chainstate; -class CBlockTreeDB; class CTxMemPool; class ChainstateManager; struct ChainTxData; diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index 99763ab97ff..76b9277e2fb 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -18,7 +18,7 @@ class BlocksdirTest(BitcoinTestFramework): def run_test(self): self.stop_node(0) - assert os.path.isdir(os.path.join(self.nodes[0].chain_path, "blocks")) + assert os.path.isdir(os.path.join(self.nodes[0].blocks_path)) assert not os.path.isdir(os.path.join(self.nodes[0].datadir, "blocks")) shutil.rmtree(self.nodes[0].datadir) initialize_datadir(self.options.tmpdir, 0, self.chain) @@ -31,7 +31,7 @@ class BlocksdirTest(BitcoinTestFramework): self.log.info("mining blocks..") self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) assert os.path.isfile(os.path.join(blocksdir_path, self.chain, "blocks", "blk00000.dat")) - assert os.path.isdir(os.path.join(self.nodes[0].chain_path, "blocks", "index")) + assert os.path.isdir(os.path.join(self.nodes[0].blocks_path, "index")) if __name__ == '__main__': diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 15dd4827ae9..4b548ef0f33 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -91,7 +91,7 @@ class PruneTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - self.prunedir = os.path.join(self.nodes[2].chain_path, 'blocks', '') + self.prunedir = os.path.join(self.nodes[2].blocks_path, '') self.connect_nodes(0, 1) self.connect_nodes(1, 2) @@ -290,7 +290,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(ret + 1, node.getblockchaininfo()['pruneheight']) def has_block(index): - return os.path.isfile(os.path.join(self.nodes[node_number].chain_path, "blocks", f"blk{index:05}.dat")) + return os.path.isfile(os.path.join(self.nodes[node_number].blocks_path, f"blk{index:05}.dat")) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py index 572e12df138..939271b3851 100755 --- a/test/functional/feature_txindex_compatibility.py +++ b/test/functional/feature_txindex_compatibility.py @@ -7,7 +7,6 @@ Previous releases are required by this test, see test/README.md. """ -import os import shutil from test_framework.test_framework import BitcoinTestFramework @@ -55,10 +54,6 @@ class TxindexCompatibilityTest(BitcoinTestFramework): drop_index_chain_dir = self.nodes[1].chain_path shutil.rmtree(drop_index_chain_dir) shutil.copytree(legacy_chain_dir, drop_index_chain_dir) - self.nodes[1].assert_start_raises_init_error( - extra_args=["-txindex"], - expected_msg="Error: The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.", - ) # Build txindex from scratch and check there is no error this time self.start_node(1, extra_args=["-txindex"]) self.wait_until(lambda: self.nodes[1].getindexinfo()["txindex"]["synced"] == True) @@ -66,12 +61,6 @@ class TxindexCompatibilityTest(BitcoinTestFramework): self.stop_nodes() - self.log.info("Check migrated txindex cannot be read by legacy node") - err_msg = f": You need to rebuild the database using -reindex to change -txindex.{os.linesep}Please restart with -reindex or -reindex-chainstate to recover." - shutil.rmtree(legacy_chain_dir) - shutil.copytree(drop_index_chain_dir, legacy_chain_dir) - self.nodes[0].assert_start_raises_init_error(extra_args=["-txindex"], expected_msg=err_msg) - if __name__ == "__main__": TxindexCompatibilityTest().main() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 801a068b49e..9f6f54c7a60 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -221,7 +221,7 @@ class WalletBackupTest(BitcoinTestFramework): self.erase_three() #start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].chain_path, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].blocks_path)) shutil.rmtree(os.path.join(self.nodes[2].chain_path, 'chainstate')) self.start_three(["-nowallet"]) diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 0fb0d7ea974..62f8301c16e 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -87,7 +87,7 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) # we need to delete the complete chain directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used - shutil.rmtree(os.path.join(self.nodes[1].chain_path, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].blocks_path)) shutil.rmtree(os.path.join(self.nodes[1].chain_path, "chainstate")) shutil.copyfile( os.path.join(self.nodes[1].datadir, "hd.bak"), @@ -115,7 +115,7 @@ class WalletHDTest(BitcoinTestFramework): # Try a RPC based rescan self.stop_node(1) - shutil.rmtree(os.path.join(self.nodes[1].chain_path, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].blocks_path)) shutil.rmtree(os.path.join(self.nodes[1].chain_path, "chainstate")) shutil.copyfile( os.path.join(self.nodes[1].datadir, "hd.bak"), diff --git a/test/functional/wallet_pruning.py b/test/functional/wallet_pruning.py index 9e6061287cc..06bd992da71 100755 --- a/test/functional/wallet_pruning.py +++ b/test/functional/wallet_pruning.py @@ -106,7 +106,7 @@ class WalletPruningTest(BitcoinTestFramework): def has_block(self, block_index): """Checks if the pruned node has the specific blk0000*.dat file""" - return os.path.isfile(os.path.join(self.nodes[1].chain_path, "blocks", f"blk{block_index:05}.dat")) + return os.path.isfile(os.path.join(self.nodes[1].blocks_path, f"blk{block_index:05}.dat")) def create_wallet(self, wallet_name, *, unload=False): """Creates and dumps a wallet on the non-pruned node0 to be later import by the pruned node"""