From 168360f4ae47cbfdb30a2cc4704435bc67e12f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C5=91rinc?= Date: Sat, 6 Sep 2025 21:25:13 -0700 Subject: [PATCH] coins: warn on oversized -dbcache Oversized allocations can cause out-of-memory errors or [heavy swapping](https://github.com/getumbrel/umbrel-os/issues/64#issuecomment-663637321), [grinding the system to a halt](https://x.com/murchandamus/status/1964432335849607224). `LogOversizedDbCache()` now emits a startup warning if the configured `-dbcache` exceeds a cap derived from system RAM, using the same parsing/clamping as cache sizing via CalculateDbCacheBytes(). This isn't meant as a recommended setting, rather a likely upper limit. Note that we're not modifying the set value, just issuing a warning. Also note that the 75% calculation is rounded for the last two numbers since we have to divide first before multiplying, otherwise we wouldn't stay inside size_t on 32-bit systems - and this was simpler than casting back and forth. We could have chosen the remaining free memory for the warning (e.g. warn if free memory is less than 1 GiB), but this is just a heuristic, we assumed that on systems with a lot of memory, other processes are also running, while memory constrained ones run only Core. If total RAM < 2 GiB, cap is `DEFAULT_DB_CACHE` (`450 MiB`), otherwise it's 75% of total RAM. The threshold is chosen to be close to values commonly used in [raspiblitz](https://github.com/raspiblitz/raspiblitz/blob/dev/home.admin/_provision.setup.sh#L98-L115) for common setups: | Total RAM | `dbcache` (MiB) | raspiblitz % | proposed cap (MiB) | |----------:|----------------:|-------------:|-------------------:| | 1 GiB | 512 | 50.0% | 450* | | 2 GiB | 1536 | 75.0% | 1536 | | 4 GiB | 2560 | 62.5% | 3072 | | 8 GiB | 4096 | 50.0% | 6144 | | 16 GiB | 4096 | 25.0% | 12288 | | 32 GiB | 4096 | 12.5% | 24576 | [Umbrel issues](https://github.com/getumbrel/umbrel-os/issues/64#issuecomment-663816367) also mention 75% being the upper limit. Starting `bitcoind` on an 8 GiB rpi4b with a dbcache of 7 GiB: > ./build/bin/bitcoind -dbcache=7000 warns now as follows: ``` 2025-09-07T17:24:29Z [warning] A 7000 MiB dbcache may be too large for a system memory of only 7800 MiB. 2025-09-07T17:24:29Z Cache configuration: 2025-09-07T17:24:29Z * Using 2.0 MiB for block index database 2025-09-07T17:24:29Z * Using 8.0 MiB for chain state database 2025-09-07T17:24:29Z * Using 6990.0 MiB for in-memory UTXO set (plus up to 286.1 MiB of unused mempool space) ``` Besides the [godbolt](https://godbolt.org/z/EPsaE3xTj) reproducers for the new total memory method, we also tested the warnings manually on: - [x] Apple M4 Max, macOS 15.6.1 - [x] Intel Core i9-9900K, Ubuntu 24.04.2 LTS - [x] Raspberry Pi 4 Model B, Armbian Linux 6.12.22-current-bcm2711 - [x] Intel Xeon x64, Windows 11 Home Version 24H2, OS Build 26100.4351 Co-authored-by: stickies-v Co-authored-by: Hodlinator <172445034+hodlinator@users.noreply.github.com> Co-authored-by: w0xlt --- src/init.cpp | 1 + src/node/caches.cpp | 34 +++++++++++++++++++++++++-------- src/node/caches.h | 7 +++++++ src/test/CMakeLists.txt | 1 + src/test/caches_tests.cpp | 40 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 src/test/caches_tests.cpp diff --git a/src/init.cpp b/src/init.cpp index 4ff57a34171..2f518eba14c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1767,6 +1767,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain // cache size calculations + node::LogOversizedDbCache(args); const auto [index_cache_sizes, kernel_cache_sizes] = CalculateCacheSizes(args, g_enabled_filter_types.size()); LogInfo("Cache configuration:"); diff --git a/src/node/caches.cpp b/src/node/caches.cpp index d5d69fc2044..e4a5e6d17e3 100644 --- a/src/node/caches.cpp +++ b/src/node/caches.cpp @@ -5,9 +5,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -23,16 +26,20 @@ static constexpr size_t MAX_FILTER_INDEX_CACHE{1024_MiB}; static constexpr size_t MAX_32BIT_DBCACHE{1024_MiB}; namespace node { +size_t CalculateDbCacheBytes(const ArgsManager& args) +{ + if (auto db_cache{args.GetIntArg("-dbcache")}) { + if (*db_cache < 0) db_cache = 0; + const uint64_t db_cache_bytes{SaturatingLeftShift(*db_cache, 20)}; + constexpr auto max_db_cache{sizeof(void*) == 4 ? MAX_32BIT_DBCACHE : std::numeric_limits::max()}; + return std::max(MIN_DB_CACHE, std::min(db_cache_bytes, max_db_cache)); + } + return DEFAULT_DB_CACHE; +} + CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes) { - // Convert -dbcache from MiB units to bytes. The total cache is floored by MIN_DB_CACHE and capped by max size_t value. - size_t total_cache{DEFAULT_DB_CACHE}; - if (std::optional db_cache = args.GetIntArg("-dbcache")) { - if (*db_cache < 0) db_cache = 0; - uint64_t db_cache_bytes = SaturatingLeftShift(*db_cache, 20); - constexpr auto max_db_cache{sizeof(void*) == 4 ? MAX_32BIT_DBCACHE : std::numeric_limits::max()}; - total_cache = std::max(MIN_DB_CACHE, std::min(db_cache_bytes, max_db_cache)); - } + size_t total_cache{CalculateDbCacheBytes(args)}; IndexCacheSizes index_sizes; index_sizes.tx_index = std::min(total_cache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? MAX_TX_INDEX_CACHE : 0); @@ -44,4 +51,15 @@ CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes) } return {index_sizes, kernel::CacheSizes{total_cache}}; } + +void LogOversizedDbCache(const ArgsManager& args) noexcept +{ + if (const auto total_ram{GetTotalRAM()}) { + const size_t db_cache{CalculateDbCacheBytes(args)}; + if (ShouldWarnOversizedDbCache(db_cache, *total_ram)) { + InitWarning(bilingual_str{tfm::format(_("A %zu MiB dbcache may be too large for a system memory of only %zu MiB."), + db_cache >> 20, *total_ram >> 20)}); + } + } +} } // namespace node diff --git a/src/node/caches.h b/src/node/caches.h index f24e9cc9103..c0ef41ec696 100644 --- a/src/node/caches.h +++ b/src/node/caches.h @@ -27,6 +27,13 @@ struct CacheSizes { kernel::CacheSizes kernel; }; CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0); +constexpr bool ShouldWarnOversizedDbCache(size_t dbcache, size_t total_ram) noexcept +{ + const size_t cap{(total_ram < 2048_MiB) ? DEFAULT_DB_CACHE : (total_ram / 100) * 75}; + return dbcache > cap; +} + +void LogOversizedDbCache(const ArgsManager& args) noexcept; } // namespace node #endif // BITCOIN_NODE_CACHES_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a818dba7205..c963afff351 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable(test_bitcoin blockmanager_tests.cpp bloom_tests.cpp bswap_tests.cpp + caches_tests.cpp chainstate_write_tests.cpp checkqueue_tests.cpp cluster_linearize_tests.cpp diff --git a/src/test/caches_tests.cpp b/src/test/caches_tests.cpp new file mode 100644 index 00000000000..3086118fc7e --- /dev/null +++ b/src/test/caches_tests.cpp @@ -0,0 +1,40 @@ +#include +#include + +#include + +using namespace node; + +BOOST_AUTO_TEST_SUITE(caches_tests) + +BOOST_AUTO_TEST_CASE(oversized_dbcache_warning) +{ + // memory restricted setup - cap is DEFAULT_DB_CACHE (450 MiB) + BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/4_MiB, /*total_ram=*/1024_MiB)); // Under cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/512_MiB, /*total_ram=*/1024_MiB)); // At cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/1500_MiB, /*total_ram=*/1024_MiB)); // Over cap + + // 2 GiB RAM - cap is 75% + BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/1500_MiB, /*total_ram=*/2048_MiB)); // Under cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/1600_MiB, /*total_ram=*/2048_MiB)); // Over cap + + if constexpr (SIZE_MAX == UINT64_MAX) { + // 4 GiB RAM - cap is 75% + BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/2500_MiB, /*total_ram=*/4096_MiB)); // Under cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/3500_MiB, /*total_ram=*/4096_MiB)); // Over cap + + // 8 GiB RAM - cap is 75% + BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/6000_MiB, /*total_ram=*/8192_MiB)); // Under cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/7000_MiB, /*total_ram=*/8192_MiB)); // Over cap + + // 16 GiB RAM - cap is 75% + BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/10'000_MiB, /*total_ram=*/16384_MiB)); // Under cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/15'000_MiB, /*total_ram=*/16384_MiB)); // Over cap + + // 32 GiB RAM - cap is 75% + BOOST_CHECK(!ShouldWarnOversizedDbCache(/*dbcache=*/20'000_MiB, /*total_ram=*/32768_MiB)); // Under cap + BOOST_CHECK( ShouldWarnOversizedDbCache(/*dbcache=*/30'000_MiB, /*total_ram=*/32768_MiB)); // Over cap + } +} + +BOOST_AUTO_TEST_SUITE_END()