From a747ca1f516e7ec73758c6017e2eca5635ab2b74 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Sat, 22 Jun 2024 21:08:10 +0200 Subject: [PATCH] kernel: Add chainstate load options for in-memory dbs in C header This allows a user to run the kernel without creating on-disk files for the block tree and chainstate indexes. This is potentially useful in scenarios where the user needs to do some ephemeral validation operations. One specific use case is when linearizing the blocks on disk. The block files store blocks out of order, so a program may utilize the library and its header to read the blocks with one chainstate manager, and then write them back in order, and without orphans, with another chainstate maanger. To save disk resources and if the indexes are not required once done, it may be beneficial to keep the indexes in memory for the chainstate manager that writes the blocks back again. --- src/kernel/bitcoinkernel.cpp | 18 ++++++++++++++ src/kernel/bitcoinkernel.h | 20 ++++++++++++++++ src/kernel/bitcoinkernel_wrapper.h | 10 ++++++++ src/test/kernel/test_kernel.cpp | 38 ++++++++++++++++++++++++++---- 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/kernel/bitcoinkernel.cpp b/src/kernel/bitcoinkernel.cpp index 23ae8128227..5203221d747 100644 --- a/src/kernel/bitcoinkernel.cpp +++ b/src/kernel/bitcoinkernel.cpp @@ -719,6 +719,24 @@ int btck_chainstate_manager_options_set_wipe_dbs(btck_ChainstateManagerOptions* return 0; } +void btck_chainstate_manager_options_update_block_tree_db_in_memory( + btck_ChainstateManagerOptions* chainman_opts, + int block_tree_db_in_memory) +{ + auto& opts{btck_ChainstateManagerOptions::get(chainman_opts)}; + LOCK(opts.m_mutex); + opts.m_blockman_options.block_tree_db_params.memory_only = block_tree_db_in_memory == 1; +} + +void btck_chainstate_manager_options_update_chainstate_db_in_memory( + btck_ChainstateManagerOptions* chainman_opts, + int chainstate_db_in_memory) +{ + auto& opts{btck_ChainstateManagerOptions::get(chainman_opts)}; + LOCK(opts.m_mutex); + opts.m_chainstate_load_options.coins_db_in_memory = chainstate_db_in_memory == 1; +} + btck_ChainstateManager* btck_chainstate_manager_create( const btck_ChainstateManagerOptions* chainman_opts) { diff --git a/src/kernel/bitcoinkernel.h b/src/kernel/bitcoinkernel.h index 49995960a17..c7c51ae4d6b 100644 --- a/src/kernel/bitcoinkernel.h +++ b/src/kernel/bitcoinkernel.h @@ -759,6 +759,26 @@ BITCOINKERNEL_API int BITCOINKERNEL_WARN_UNUSED_RESULT btck_chainstate_manager_o int wipe_block_tree_db, int wipe_chainstate_db) BITCOINKERNEL_ARG_NONNULL(1); +/** + * @brief Sets block tree db in memory in the options. + * + * @param[in] chainstate_manager_options Non-null, created by @ref btck_chainstate_manager_options_create. + * @param[in] block_tree_db_in_memory Set block tree db in memory. + */ +BITCOINKERNEL_API void btck_chainstate_manager_options_update_block_tree_db_in_memory( + btck_ChainstateManagerOptions* chainstate_manager_options, + int block_tree_db_in_memory) BITCOINKERNEL_ARG_NONNULL(1); + +/** + * @brief Sets chainstate db in memory in the options. + * + * @param[in] chainstate_manager_options Non-null, created by @ref btck_chainstate_manager_options_create. + * @param[in] chainstate_db_in_memory Set chainstate db in memory. + */ +BITCOINKERNEL_API void btck_chainstate_manager_options_update_chainstate_db_in_memory( + btck_ChainstateManagerOptions* chainstate_manager_options, + int chainstate_db_in_memory) BITCOINKERNEL_ARG_NONNULL(1); + /** * Destroy the chainstate manager options. */ diff --git a/src/kernel/bitcoinkernel_wrapper.h b/src/kernel/bitcoinkernel_wrapper.h index 15186bbdf39..c3b3e6905ee 100644 --- a/src/kernel/bitcoinkernel_wrapper.h +++ b/src/kernel/bitcoinkernel_wrapper.h @@ -670,6 +670,16 @@ public: { return btck_chainstate_manager_options_set_wipe_dbs(get(), wipe_block_tree, wipe_chainstate) == 0; } + + void UpdateBlockTreeDbInMemory(bool block_tree_db_in_memory) + { + btck_chainstate_manager_options_update_block_tree_db_in_memory(get(), block_tree_db_in_memory); + } + + void UpdateChainstateDbInMemory(bool chainstate_db_in_memory) + { + btck_chainstate_manager_options_update_chainstate_db_in_memory(get(), chainstate_db_in_memory); + } }; class ChainMan : UniqueHandle diff --git a/src/test/kernel/test_kernel.cpp b/src/test/kernel/test_kernel.cpp index 38879f71bb7..553d68f502a 100644 --- a/src/test/kernel/test_kernel.cpp +++ b/src/test/kernel/test_kernel.cpp @@ -524,6 +524,8 @@ BOOST_AUTO_TEST_CASE(btck_chainman_tests) std::unique_ptr create_chainman(TestDirectory& test_directory, bool reindex, bool wipe_chainstate, + bool block_tree_db_in_memory, + bool chainstate_db_in_memory, Context& context) { ChainstateManagerOptions chainman_opts{context, test_directory.m_directory.string(), (test_directory.m_directory / "blocks").string()}; @@ -534,6 +536,12 @@ std::unique_ptr create_chainman(TestDirectory& test_directory, if (wipe_chainstate) { chainman_opts.SetWipeDbs(/*wipe_block_tree=*/false, /*wipe_chainstate=*/wipe_chainstate); } + if (block_tree_db_in_memory) { + chainman_opts.UpdateBlockTreeDbInMemory(block_tree_db_in_memory); + } + if (chainstate_db_in_memory) { + chainman_opts.UpdateChainstateDbInMemory(chainstate_db_in_memory); + } auto chainman{std::make_unique(context, chainman_opts)}; return chainman; @@ -543,21 +551,21 @@ void chainman_reindex_test(TestDirectory& test_directory) { auto notifications{std::make_shared()}; auto context{create_context(notifications, ChainType::MAINNET)}; - auto chainman{create_chainman(test_directory, true, false, context)}; + auto chainman{create_chainman(test_directory, true, false, false, false, context)}; } void chainman_reindex_chainstate_test(TestDirectory& test_directory) { auto notifications{std::make_shared()}; auto context{create_context(notifications, ChainType::MAINNET)}; - auto chainman{create_chainman(test_directory, false, true, context)}; + auto chainman{create_chainman(test_directory, false, true, false, false, context)}; } void chainman_mainnet_validation_test(TestDirectory& test_directory) { auto notifications{std::make_shared()}; auto context{create_context(notifications, ChainType::MAINNET)}; - auto chainman{create_chainman(test_directory, false, false, context)}; + auto chainman{create_chainman(test_directory, false, false, false, false, context)}; { // Process an invalid block @@ -601,6 +609,26 @@ BOOST_AUTO_TEST_CASE(btck_chainman_mainnet_tests) chainman_reindex_chainstate_test(test_directory); } +BOOST_AUTO_TEST_CASE(btck_chainman_in_memory_tests) +{ + auto in_memory_test_directory{TestDirectory{"in-memory_test_bitcoin_kernel"}}; + + auto notifications{std::make_shared()}; + auto context{create_context(notifications, ChainType::REGTEST)}; + auto chainman{create_chainman(in_memory_test_directory, false, false, true, true, context)}; + + for (auto& raw_block : REGTEST_BLOCK_DATA) { + Block block{hex_string_to_byte_vec(raw_block)}; + bool new_block{false}; + chainman->ProcessBlock(block, &new_block); + BOOST_CHECK(new_block); + } + + BOOST_CHECK(std::filesystem::exists(in_memory_test_directory.m_directory / "blocks")); + BOOST_CHECK(!std::filesystem::exists(in_memory_test_directory.m_directory / "blocks" / "index")); + BOOST_CHECK(!std::filesystem::exists(in_memory_test_directory.m_directory / "chainstate")); +} + BOOST_AUTO_TEST_CASE(btck_chainman_regtest_tests) { auto test_directory{TestDirectory{"regtest_test_bitcoin_kernel"}}; @@ -614,7 +642,7 @@ BOOST_AUTO_TEST_CASE(btck_chainman_regtest_tests) const size_t mid{REGTEST_BLOCK_DATA.size() / 2}; { - auto chainman{create_chainman(test_directory, false, false, context)}; + auto chainman{create_chainman(test_directory, false, false, false, false, context)}; for (size_t i{0}; i < mid; i++) { Block block{hex_string_to_byte_vec(REGTEST_BLOCK_DATA[i])}; bool new_block{false}; @@ -623,7 +651,7 @@ BOOST_AUTO_TEST_CASE(btck_chainman_regtest_tests) } } - auto chainman{create_chainman(test_directory, false, false, context)}; + auto chainman{create_chainman(test_directory, false, false, false, false, context)}; for (size_t i{mid}; i < REGTEST_BLOCK_DATA.size(); i++) { Block block{hex_string_to_byte_vec(REGTEST_BLOCK_DATA[i])};