From fa359255fe6b4de5f26784bfc147dbfb58bef116 Mon Sep 17 00:00:00 2001
From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz>
Date: Fri, 30 Jun 2023 12:42:47 +0200
Subject: [PATCH] Add -blocksxor boolean option
---
doc/files.md | 3 ++-
doc/release-notes-28052.md | 6 +++++
src/init.cpp | 13 +++++++++-
src/kernel/blockmanager_opts.h | 3 +++
src/node/blockmanager_args.cpp | 1 +
src/node/blockstorage.cpp | 37 +++++++++++++++++++++++++++-
test/functional/feature_loadblock.py | 4 +++
test/functional/feature_reindex.py | 18 +++++++++++---
8 files changed, 78 insertions(+), 7 deletions(-)
create mode 100644 doc/release-notes-28052.md
diff --git a/doc/files.md b/doc/files.md
index 03e52f02c9..5c5a61bd56 100644
--- a/doc/files.md
+++ b/doc/files.md
@@ -47,8 +47,9 @@ Subdirectory | File(s) | Description
-------------------|-----------------------|------------
`blocks/` | | Blocks directory; can be specified by `-blocksdir` option (except for `blocks/index/`)
`blocks/index/` | LevelDB database | Block index; `-blocksdir` option does not affect this path
-`blocks/` | `blkNNNNN.dat`[\[2\]](#note2) | Actual Bitcoin blocks (in network format, dumped in raw on disk, 128 MiB per file)
+`blocks/` | `blkNNNNN.dat`[\[2\]](#note2) | Actual Bitcoin blocks (dumped in network format, 128 MiB per file)
`blocks/` | `revNNNNN.dat`[\[2\]](#note2) | Block undo data (custom format)
+`blocks/` | `xor.dat` | Rolling XOR pattern for block and undo data files
`chainstate/` | LevelDB database | Blockchain state (a compact representation of all currently unspent transaction outputs (UTXOs) and metadata about the transactions they are from)
`indexes/txindex/` | LevelDB database | Transaction index; *optional*, used if `-txindex=1`
`indexes/blockfilter/basic/db/` | LevelDB database | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
diff --git a/doc/release-notes-28052.md b/doc/release-notes-28052.md
new file mode 100644
index 0000000000..386f0cee5f
--- /dev/null
+++ b/doc/release-notes-28052.md
@@ -0,0 +1,6 @@
+Blockstorage
+============
+
+Block files are now XOR'd by default with a key stored in the blocksdir.
+Previous releases of Bitcoin Core or previous external software will not be able to read the blocksdir with a non-zero XOR-key.
+Refer to the `-blocksxor` help for more details.
diff --git a/src/init.cpp b/src/init.cpp
index 9e570d6128..085b7f976a 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -469,6 +469,13 @@ void SetupServerArgs(ArgsManager& argsman)
#endif
argsman.AddArg("-assumevalid=", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), signetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
argsman.AddArg("-blocksdir=", "Specify directory to hold blocks subdirectory for *.dat files (default: )", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+ argsman.AddArg("-blocksxor",
+ strprintf("Whether an XOR-key applies to blocksdir *.dat files. "
+ "The created XOR-key will be zeros for an existing blocksdir or when `-blocksxor=0` is "
+ "set, and random for a freshly initialized blocksdir. "
+ "(default: %u)",
+ kernel::DEFAULT_XOR_BLOCKSDIR),
+ ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
#if HAVE_SYSTEM
argsman.AddArg("-blocknotify=", "Execute command when the best block changes (%s in cmd is replaced by block hash)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -1534,7 +1541,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
}
LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024));
- node.chainman = std::make_unique(*Assert(node.shutdown), chainman_opts, blockman_opts);
+ try {
+ node.chainman = std::make_unique(*Assert(node.shutdown), chainman_opts, blockman_opts);
+ } catch (std::exception& e) {
+ return InitError(strprintf(Untranslated("Failed to initialize ChainstateManager: %s"), e.what()));
+ }
ChainstateManager& chainman = *node.chainman;
// This is defined and set here instead of inline in validation.h to avoid a hard
diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h
index deeba7e318..261ec3be58 100644
--- a/src/kernel/blockmanager_opts.h
+++ b/src/kernel/blockmanager_opts.h
@@ -14,12 +14,15 @@ class CChainParams;
namespace kernel {
+static constexpr bool DEFAULT_XOR_BLOCKSDIR{true};
+
/**
* An options struct for `BlockManager`, more ergonomically referred to as
* `BlockManager::Options` due to the using-declaration in `BlockManager`.
*/
struct BlockManagerOpts {
const CChainParams& chainparams;
+ bool use_xor{DEFAULT_XOR_BLOCKSDIR};
uint64_t prune_target{0};
bool fast_prune{false};
const fs::path blocks_dir;
diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp
index fa76566652..0fc4e1646a 100644
--- a/src/node/blockmanager_args.cpp
+++ b/src/node/blockmanager_args.cpp
@@ -16,6 +16,7 @@
namespace node {
util::Result ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts)
{
+ if (auto value{args.GetBoolArg("-blocksxor")}) opts.use_xor = *value;
// block pruning; get the amount of disk space (in MiB) to allot for block & undo files
int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)};
if (nPruneArg < 0) {
diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp
index 338452c23f..8db5cbc9fd 100644
--- a/src/node/blockstorage.cpp
+++ b/src/node/blockstorage.cpp
@@ -19,6 +19,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -1144,9 +1145,43 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight)
return blockPos;
}
+static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
+{
+ // Bytes are serialized without length indicator, so this is also the exact
+ // size of the XOR-key file.
+ std::array xor_key{};
+
+ if (opts.use_xor && fs::is_empty(opts.blocks_dir)) {
+ // Only use random fresh key when the boolean option is set and on the
+ // very first start of the program.
+ FastRandomContext{}.fillrand(xor_key);
+ }
+
+ const fs::path xor_key_path{opts.blocks_dir / "xor.dat"};
+ if (fs::exists(xor_key_path)) {
+ // A pre-existing xor key file has priority.
+ AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "rb")};
+ xor_key_file >> xor_key;
+ } else {
+ // Create initial or missing xor key file
+ AutoFile xor_key_file{fsbridge::fopen(xor_key_path, "wbx")};
+ xor_key_file << xor_key;
+ }
+ // If the user disabled the key, it must be zero.
+ if (!opts.use_xor && xor_key != decltype(xor_key){}) {
+ throw std::runtime_error{
+ strprintf("The blocksdir XOR-key can not be disabled when a random key was already stored! "
+ "Stored key: '%s', stored path: '%s'.",
+ HexStr(xor_key), fs::PathToString(xor_key_path)),
+ };
+ }
+ LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
+ return std::vector{xor_key.begin(), xor_key.end()};
+}
+
BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
: m_prune_mode{opts.prune_target > 0},
- m_xor_key{},
+ m_xor_key{InitBlocksdirXorKey(opts)},
m_opts{std::move(opts)},
m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py
index 1519c132b9..824308ac95 100755
--- a/test/functional/feature_loadblock.py
+++ b/test/functional/feature_loadblock.py
@@ -26,6 +26,10 @@ class LoadblockTest(BitcoinTestFramework):
self.setup_clean_chain = True
self.num_nodes = 2
self.supports_cli = False
+ self.extra_args = [
+ ["-blocksxor=0"], # TODO: The linearize scripts should be adjusted to apply any XOR
+ [],
+ ]
def run_test(self):
self.nodes[1].setnetworkactive(state=False)
diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py
index 50a9ae1c06..504b3dfff4 100755
--- a/test/functional/feature_reindex.py
+++ b/test/functional/feature_reindex.py
@@ -39,9 +39,19 @@ class ReindexTest(BitcoinTestFramework):
# we're generating them rather than getting them from peers), so to
# test out-of-order handling, swap blocks 1 and 2 on disk.
blk0 = self.nodes[0].blocks_path / "blk00000.dat"
+ with open(self.nodes[0].blocks_path / "xor.dat", "rb") as xor_f:
+ NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size()
+ xor_dat = xor_f.read(NUM_XOR_BYTES)
+
+ def util_xor(data, key, *, offset):
+ data = bytearray(data)
+ for i in range(len(data)):
+ data[i] ^= key[(i + offset) % len(key)]
+ return bytes(data)
+
with open(blk0, 'r+b') as bf:
# Read at least the first few blocks (including genesis)
- b = bf.read(2000)
+ b = util_xor(bf.read(2000), xor_dat, offset=0)
# Find the offsets of blocks 2, 3, and 4 (the first 3 blocks beyond genesis)
# by searching for the regtest marker bytes (see pchMessageStart).
@@ -55,12 +65,12 @@ class ReindexTest(BitcoinTestFramework):
b4_start = find_block(b, b3_start)
# Blocks 2 and 3 should be the same size.
- assert_equal(b3_start-b2_start, b4_start-b3_start)
+ assert_equal(b3_start - b2_start, b4_start - b3_start)
# Swap the second and third blocks (don't disturb the genesis block).
bf.seek(b2_start)
- bf.write(b[b3_start:b4_start])
- bf.write(b[b2_start:b3_start])
+ bf.write(util_xor(b[b3_start:b4_start], xor_dat, offset=b2_start))
+ bf.write(util_xor(b[b2_start:b3_start], xor_dat, offset=b3_start))
# The reindexing code should detect and accommodate out of order blocks.
with self.nodes[0].assert_debug_log([