diff --git a/doc/files.md b/doc/files.md index 03e52f02c92..5c5a61bd56e 100644 --- a/doc/files.md +++ b/doc/files.md @@ -47,8 +47,9 @@ Subdirectory | File(s) | Description -------------------|-----------------------|------------ `blocks/` | | Blocks directory; can be specified by `-blocksdir` option (except for `blocks/index/`) `blocks/index/` | LevelDB database | Block index; `-blocksdir` option does not affect this path -`blocks/` | `blkNNNNN.dat`[\[2\]](#note2) | Actual Bitcoin blocks (in network format, dumped in raw on disk, 128 MiB per file) +`blocks/` | `blkNNNNN.dat`[\[2\]](#note2) | Actual Bitcoin blocks (dumped in network format, 128 MiB per file) `blocks/` | `revNNNNN.dat`[\[2\]](#note2) | Block undo data (custom format) +`blocks/` | `xor.dat` | Rolling XOR pattern for block and undo data files `chainstate/` | LevelDB database | Blockchain state (a compact representation of all currently unspent transaction outputs (UTXOs) and metadata about the transactions they are from) `indexes/txindex/` | LevelDB database | Transaction index; *optional*, used if `-txindex=1` `indexes/blockfilter/basic/db/` | LevelDB database | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic` diff --git a/doc/release-notes-28052.md b/doc/release-notes-28052.md new file mode 100644 index 00000000000..386f0cee5f0 --- /dev/null +++ b/doc/release-notes-28052.md @@ -0,0 +1,6 @@ +Blockstorage +============ + +Block files are now XOR'd by default with a key stored in the blocksdir. +Previous releases of Bitcoin Core or previous external software will not be able to read the blocksdir with a non-zero XOR-key. +Refer to the `-blocksxor` help for more details. diff --git a/src/init.cpp b/src/init.cpp index 9e570d61280..085b7f976a9 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -469,6 +469,13 @@ void SetupServerArgs(ArgsManager& argsman) #endif argsman.AddArg("-assumevalid=", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), signetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksdir=", "Specify directory to hold blocks subdirectory for *.dat files (default: )", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-blocksxor", + strprintf("Whether an XOR-key applies to blocksdir *.dat files. " + "The created XOR-key will be zeros for an existing blocksdir or when `-blocksxor=0` is " + "set, and random for a freshly initialized blocksdir. " + "(default: %u)", + kernel::DEFAULT_XOR_BLOCKSDIR), + ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); #if HAVE_SYSTEM argsman.AddArg("-blocknotify=", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -1534,7 +1541,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); - node.chainman = std::make_unique(*Assert(node.shutdown), chainman_opts, blockman_opts); + try { + node.chainman = std::make_unique(*Assert(node.shutdown), chainman_opts, blockman_opts); + } catch (std::exception& e) { + return InitError(strprintf(Untranslated("Failed to initialize ChainstateManager: %s"), e.what())); + } ChainstateManager& chainman = *node.chainman; // This is defined and set here instead of inline in validation.h to avoid a hard diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index deeba7e318b..261ec3be582 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -14,12 +14,15 @@ class CChainParams; namespace kernel { +static constexpr bool DEFAULT_XOR_BLOCKSDIR{true}; + /** * An options struct for `BlockManager`, more ergonomically referred to as * `BlockManager::Options` due to the using-declaration in `BlockManager`. */ struct BlockManagerOpts { const CChainParams& chainparams; + bool use_xor{DEFAULT_XOR_BLOCKSDIR}; uint64_t prune_target{0}; bool fast_prune{false}; const fs::path blocks_dir; diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index fa765666525..0fc4e1646a1 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -16,6 +16,7 @@ namespace node { util::Result ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) { + if (auto value{args.GetBoolArg("-blocksxor")}) opts.use_xor = *value; // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)}; if (nPruneArg < 0) { diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index bfb11e9fcd4..4cff587d51b 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -818,13 +819,13 @@ void BlockManager::UnlinkPrunedFiles(const std::set& setFilesToPrune) const AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { - return AutoFile{m_block_file_seq.Open(pos, fReadOnly)}; + return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key}; } /** Open an undo file (rev?????.dat) */ AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { - return AutoFile{m_undo_file_seq.Open(pos, fReadOnly)}; + return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key}; } fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const @@ -1144,6 +1145,54 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight) return blockPos; } +static auto InitBlocksdirXorKey(const BlockManager::Options& opts) +{ + // Bytes are serialized without length indicator, so this is also the exact + // size of the XOR-key file. + std::array xor_key{}; + + if (opts.use_xor && fs::is_empty(opts.blocks_dir)) { + // Only use random fresh key when the boolean option is set and on the + // very first start of the program. + FastRandomContext{}.fillrand(xor_key); + } + + const fs::path xor_key_path{opts.blocks_dir / "xor.dat"}; + if (fs::exists(xor_key_path)) { + // A pre-existing xor key file has priority. + AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")}; + xor_key_file >> xor_key; + } else { + // Create initial or missing xor key file + AutoFile xor_key_file{fsbridge::fopen(xor_key_path, +#ifdef __MINGW64__ + "wb" // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210 +#else + "wbx" +#endif + )}; + xor_key_file << xor_key; + } + // If the user disabled the key, it must be zero. + if (!opts.use_xor && xor_key != decltype(xor_key){}) { + throw std::runtime_error{ + strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! " + "Stored key: '%s', stored path: '%s'.", + HexStr(xor_key), fs::PathToString(xor_key_path)), + }; + } + LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key)); + return std::vector{xor_key.begin(), xor_key.end()}; +} + +BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts) + : m_prune_mode{opts.prune_target > 0}, + m_xor_key{InitBlocksdirXorKey(opts)}, + m_opts{std::move(opts)}, + m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}}, + m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}}, + m_interrupt{interrupt} {} + class ImportingNow { std::atomic& m_importing; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 8c6190cd02f..821bbf5109b 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -240,6 +240,8 @@ private: const bool m_prune_mode; + const std::vector m_xor_key; + /** Dirty block index entries. */ std::set m_dirty_blockindex; @@ -264,12 +266,7 @@ private: public: using Options = kernel::BlockManagerOpts; - explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) - : m_prune_mode{opts.prune_target > 0}, - m_opts{std::move(opts)}, - m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}}, - m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}}, - m_interrupt{interrupt} {} + explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts); const util::SignalInterrupt& m_interrupt; std::atomic m_importing{false}; diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index eed932b6d29..9296cbb41c4 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -30,8 +30,7 @@ BOOST_AUTO_TEST_CASE(xor_file) } { #ifdef __MINGW64__ - // Our usage of mingw-w64 and the msvcrt runtime does not support - // the x modifier for the _wfopen(). + // Temporary workaround for https://github.com/bitcoin/bitcoin/issues/30210 const char* mode = "wb"; #else const char* mode = "wbx"; diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 1519c132b98..824308ac956 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -26,6 +26,10 @@ class LoadblockTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 self.supports_cli = False + self.extra_args = [ + ["-blocksxor=0"], # TODO: The linearize scripts should be adjusted to apply any XOR + [], + ] def run_test(self): self.nodes[1].setnetworkactive(state=False) diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index 50a9ae1c060..504b3dfff44 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -39,9 +39,19 @@ class ReindexTest(BitcoinTestFramework): # we're generating them rather than getting them from peers), so to # test out-of-order handling, swap blocks 1 and 2 on disk. blk0 = self.nodes[0].blocks_path / "blk00000.dat" + with open(self.nodes[0].blocks_path / "xor.dat", "rb") as xor_f: + NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size() + xor_dat = xor_f.read(NUM_XOR_BYTES) + + def util_xor(data, key, *, offset): + data = bytearray(data) + for i in range(len(data)): + data[i] ^= key[(i + offset) % len(key)] + return bytes(data) + with open(blk0, 'r+b') as bf: # Read at least the first few blocks (including genesis) - b = bf.read(2000) + b = util_xor(bf.read(2000), xor_dat, offset=0) # Find the offsets of blocks 2, 3, and 4 (the first 3 blocks beyond genesis) # by searching for the regtest marker bytes (see pchMessageStart). @@ -55,12 +65,12 @@ class ReindexTest(BitcoinTestFramework): b4_start = find_block(b, b3_start) # Blocks 2 and 3 should be the same size. - assert_equal(b3_start-b2_start, b4_start-b3_start) + assert_equal(b3_start - b2_start, b4_start - b3_start) # Swap the second and third blocks (don't disturb the genesis block). bf.seek(b2_start) - bf.write(b[b3_start:b4_start]) - bf.write(b[b2_start:b3_start]) + bf.write(util_xor(b[b3_start:b4_start], xor_dat, offset=b2_start)) + bf.write(util_xor(b[b2_start:b3_start], xor_dat, offset=b3_start)) # The reindexing code should detect and accommodate out of order blocks. with self.nodes[0].assert_debug_log([