Merge a06a674b4361061432d20dc21f183b7eac0175b3 into 5f4422d68dc3530c353af1f87499de1c864b60ad

3301d2cbe8c3b76c97285d75fa59637cb6952d0b qa: Wait for txindex to avoid race condition (Hodlinator)
9bfb0d75ba10591cc6c9620f9fd1ecc0e55e7a48 qa: Remove unnecessary -txindex args (Hodlinator)
7ac281c19cd3d11f316dbbb3308eabf1ad4f26d6 qa: Add missing coverage of corrupt indexes (Hodlinator)

Pull request description:

  - Add synchronization in 3 places where if the Transaction Index happens to be slow, we get rare test failures when querying it for transactions (one such case experienced on Windows, prompting investigation).
  - Remove unnecessary TxIndex initialization in some tests.
  - Add some test coverage where TxIndex aspect could be tested in feature_init.py.

ACKs for top commit:
  fjahr:
    re-ACK 3301d2cbe8c3b76c97285d75fa59637cb6952d0b
  mzumsande:
    Code Review ACK 3301d2cbe8c3b76c97285d75fa59637cb6952d0b
  furszy:
    Code review ACK 3301d2cbe8c3b76c97285d75fa59637cb6952d0b
  Prabhat1308:
    Concept ACK [`3301d2c`](3301d2cbe8)

Tree-SHA512: 7c2019e38455f344856aaf6b381faafbd88d53dc88d13309deb718c1dcfbee4ccca7c7f1b66917395503a6f94c3b216a007ad432cc8b93d0309db9805f38d602

src/bench/checkblock.cpp

@@ -21,11 +21,34 @@
 #include <optional>
 #include <vector>
 
+static void SizeComputerBlock(benchmark::Bench& bench) {
+    CBlock block;
+    DataStream(benchmark::data::block413567) >> TX_WITH_WITNESS(block);
+
+    bench.unit("block").run([&] {
+        SizeComputer size_computer;
+        size_computer << TX_WITH_WITNESS(block);
+        assert(size_computer.size() == benchmark::data::block413567.size());
+    });
+}
+
+static void SerializeBlock(benchmark::Bench& bench) {
+    CBlock block;
+    DataStream(benchmark::data::block413567) >> TX_WITH_WITNESS(block);
+
+    // Create output stream and verify first serialization matches input
+    bench.unit("block").run([&] {
+        DataStream output_stream(benchmark::data::block413567.size());
+        output_stream << TX_WITH_WITNESS(block);
+        assert(output_stream.size() == benchmark::data::block413567.size());
+    });
+}
+
 // These are the two major time-sinks which happen after we have fully received
 // a block off the wire, but before we can relay the block on to peers using
 // compact block relay.
 
-static void DeserializeBlockTest(benchmark::Bench& bench)
+static void DeserializeBlockBench(benchmark::Bench& bench)
 {
     DataStream stream(benchmark::data::block413567);
     std::byte a{0};
@@ -39,26 +62,20 @@ static void DeserializeBlockTest(benchmark::Bench& bench)
     });
 }
 
-static void DeserializeAndCheckBlockTest(benchmark::Bench& bench)
+static void CheckBlockBench(benchmark::Bench& bench)
 {
-    DataStream stream(benchmark::data::block413567);
-    std::byte a{0};
-    stream.write({&a, 1}); // Prevent compaction
-
-    ArgsManager bench_args;
-    const auto chainParams = CreateChainParams(bench_args, ChainType::MAIN);
-
+    CBlock block;
+    DataStream(benchmark::data::block413567) >> TX_WITH_WITNESS(block);
+    const auto chainParams = CreateChainParams(ArgsManager{}, ChainType::MAIN);
     bench.unit("block").run([&] {
-        CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here
-        stream >> TX_WITH_WITNESS(block);
-        bool rewound = stream.Rewind(benchmark::data::block413567.size());
-        assert(rewound);
-
+        block.fChecked = block.m_checked_witness_commitment = block.m_checked_merkle_root = false; // Reset the cached state
         BlockValidationState validationState;
-        bool checked = CheckBlock(block, validationState, chainParams->GetConsensus());
-        assert(checked);
+        bool checked = CheckBlock(block, validationState, chainParams->GetConsensus(), /*fCheckPOW=*/true, /*fCheckMerkleRoot=*/true);
+        assert(checked && validationState.IsValid());
     });
 }
 
-BENCHMARK(DeserializeBlockTest, benchmark::PriorityLevel::HIGH);
-BENCHMARK(DeserializeAndCheckBlockTest, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SizeComputerBlock, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SerializeBlock, benchmark::PriorityLevel::HIGH);
+BENCHMARK(DeserializeBlockBench, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CheckBlockBench, benchmark::PriorityLevel::HIGH);

src/bench/crypto_hash.cpp

@@ -2,7 +2,6 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-
 #include <bench/bench.h>
 #include <crypto/muhash.h>
 #include <crypto/ripemd160.h>
@@ -12,9 +11,11 @@
 #include <crypto/sha512.h>
 #include <crypto/siphash.h>
 #include <random.h>
-#include <span.h>
 #include <tinyformat.h>
 #include <uint256.h>
+#include <primitives/transaction.h>
+#include <util/hasher.h>
+#include <unordered_set>
 
 #include <cstdint>
 #include <vector>
@@ -205,6 +206,98 @@ static void SipHash_32b(benchmark::Bench& bench)
     });
 }
 
+static void SaltedOutpointHasherBench_hash(benchmark::Bench& bench)
+{
+    FastRandomContext rng{/*fDeterministic=*/true};
+    constexpr size_t size{1000};
+
+    std::vector<COutPoint> outpoints(size);
+    for (auto& outpoint : outpoints) {
+        outpoint = {Txid::FromUint256(rng.rand256()), rng.rand32()};
+    }
+
+    const SaltedOutpointHasher hasher;
+    bench.batch(size).run([&] {
+        size_t result{0};
+        for (const auto& outpoint : outpoints) {
+            result ^= hasher(outpoint);
+        }
+        ankerl::nanobench::doNotOptimizeAway(result);
+    });
+}
+
+static void SaltedOutpointHasherBench_match(benchmark::Bench& bench)
+{
+    FastRandomContext rng{/*fDeterministic=*/true};
+    constexpr size_t size{1000};
+
+    std::unordered_set<COutPoint, SaltedOutpointHasher> values;
+    std::vector<COutPoint> value_vector;
+    values.reserve(size);
+    value_vector.reserve(size);
+
+    for (size_t i{0}; i < size; ++i) {
+        COutPoint outpoint{Txid::FromUint256(rng.rand256()), rng.rand32()};
+        values.emplace(outpoint);
+        value_vector.push_back(outpoint);
+        assert(values.contains(outpoint));
+    }
+
+    bench.batch(size).run([&] {
+        bool result = true;
+        for (const auto& outpoint : value_vector) {
+            result ^= values.contains(outpoint);
+        }
+        ankerl::nanobench::doNotOptimizeAway(result);
+    });
+}
+
+static void SaltedOutpointHasherBench_mismatch(benchmark::Bench& bench)
+{
+    FastRandomContext rng{/*fDeterministic=*/true};
+    constexpr size_t size{1000};
+
+    std::unordered_set<COutPoint, SaltedOutpointHasher> values;
+    std::vector<COutPoint> missing_value_vector;
+    values.reserve(size);
+    missing_value_vector.reserve(size);
+
+    for (size_t i{0}; i < size; ++i) {
+        values.emplace(Txid::FromUint256(rng.rand256()), rng.rand32());
+        COutPoint missing_outpoint{Txid::FromUint256(rng.rand256()), rng.rand32()};
+        missing_value_vector.push_back(missing_outpoint);
+        assert(!values.contains(missing_outpoint));
+    }
+
+    bench.batch(size).run([&] {
+        bool result{false};
+        for (const auto& outpoint : missing_value_vector) {
+            result ^= values.contains(outpoint);
+        }
+        ankerl::nanobench::doNotOptimizeAway(result);
+    });
+}
+
+static void SaltedOutpointHasherBench_create_set(benchmark::Bench& bench)
+{
+    FastRandomContext rng{/*fDeterministic=*/true};
+    constexpr size_t size{1000};
+
+    std::vector<COutPoint> outpoints(size);
+    for (auto& outpoint : outpoints) {
+        outpoint = {Txid::FromUint256(rng.rand256()), rng.rand32()};
+    }
+
+    bench.batch(size).run([&] {
+        std::unordered_set<COutPoint, SaltedOutpointHasher> set;
+        set.reserve(size);
+        for (const auto& outpoint : outpoints) {
+            set.emplace(outpoint);
+        }
+        ankerl::nanobench::doNotOptimizeAway(set.size());
+    });
+}
+
 static void MuHash(benchmark::Bench& bench)
 {
     MuHash3072 acc;
@@ -276,6 +369,10 @@ BENCHMARK(SHA256_32b_SSE4, benchmark::PriorityLevel::HIGH);
 BENCHMARK(SHA256_32b_AVX2, benchmark::PriorityLevel::HIGH);
 BENCHMARK(SHA256_32b_SHANI, benchmark::PriorityLevel::HIGH);
 BENCHMARK(SipHash_32b, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SaltedOutpointHasherBench_hash, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SaltedOutpointHasherBench_match, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SaltedOutpointHasherBench_mismatch, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SaltedOutpointHasherBench_create_set, benchmark::PriorityLevel::HIGH);
 BENCHMARK(SHA256D64_1024_STANDARD, benchmark::PriorityLevel::HIGH);
 BENCHMARK(SHA256D64_1024_SSE4, benchmark::PriorityLevel::HIGH);
 BENCHMARK(SHA256D64_1024_AVX2, benchmark::PriorityLevel::HIGH);

src/bench/mempool_stress.cpp

@@ -13,10 +13,19 @@
 #include <test/util/txmempool.h>
 #include <txmempool.h>
 #include <validation.h>
+#include <bench/data/block413567.raw.h>
+#include <node/context.h>
+#include <node/miner.h>
+#include <primitives/block.h>
+#include <test/util/script.h>
+#include <util/check.h>
 
+#include <array>
+#include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <streams.h>
 #include <vector>
 
 class CCoinsViewCache;
@@ -126,5 +135,53 @@ static void MempoolCheck(benchmark::Bench& bench)
     });
 }
 
+static void ProcessTransactionBench(benchmark::Bench& bench)
+{
+    const auto testing_setup{MakeNoLogFileContext<const TestingSetup>()};
+    CTxMemPool& pool{*Assert(testing_setup->m_node.mempool)};
+    ChainstateManager& chainman{*testing_setup->m_node.chainman};
+
+    CBlock block;
+    DataStream(benchmark::data::block413567) >> TX_WITH_WITNESS(block);
+
+    std::vector<CTransactionRef> txs(block.vtx.size() - 1);
+    for (size_t i{1}; i < block.vtx.size(); ++i) {
+        CMutableTransaction mtx{*block.vtx[i]};
+        for (auto& txin : mtx.vin) {
+            txin.nSequence = CTxIn::SEQUENCE_FINAL;
+            txin.scriptSig.clear();
+            txin.scriptWitness.stack = {WITNESS_STACK_ELEM_OP_TRUE};
+        }
+        txs[i - 1] = MakeTransactionRef(std::move(mtx));
+    }
+
+    CCoinsViewCache* coins_tip{nullptr};
+    size_t cached_coin_count{0};
+    {
+        LOCK(cs_main);
+        coins_tip = &chainman.ActiveChainstate().CoinsTip();
+        for (const auto& tx : txs) {
+            const Coin coin(CTxOut(2 * tx->GetValueOut(), P2WSH_OP_TRUE), 1, /*fCoinBaseIn=*/false);
+            for (const auto& in : tx->vin) {
+                coins_tip->AddCoin(in.prevout, Coin{coin}, /*possible_overwrite=*/false);
+                cached_coin_count++;
+            }
+        }
+    }
+
+    bench.batch(txs.size()).run([&] {
+        LOCK2(cs_main, pool.cs);
+        assert(coins_tip->GetCacheSize() == cached_coin_count);
+        for (const auto& tx : txs) pool.removeRecursive(*tx, MemPoolRemovalReason::REPLACED);
+        assert(pool.size() == 0);
+
+        for (const auto& tx : txs) {
+            const auto res{chainman.ProcessTransaction(tx, /*test_accept=*/true)};
+            assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID);
+        }
+    });
+}
+
 BENCHMARK(ComplexMemPool, benchmark::PriorityLevel::HIGH);
 BENCHMARK(MempoolCheck, benchmark::PriorityLevel::HIGH);
+BENCHMARK(ProcessTransactionBench, benchmark::PriorityLevel::HIGH);

src/bench/xor.cpp

@@ -7,17 +7,23 @@
 #include <span.h>
 #include <streams.h>
 
+#include <cmath>
 #include <cstddef>
+#include <map>
 #include <vector>
 
 static void Xor(benchmark::Bench& bench)
 {
-    FastRandomContext frc{/*fDeterministic=*/true};
-    auto data{frc.randbytes<std::byte>(1024)};
-    auto key{frc.randbytes<std::byte>(31)};
+    FastRandomContext rng{/*fDeterministic=*/true};
+    auto test_data{rng.randbytes<std::byte>(1 << 20)};
 
-    bench.batch(data.size()).unit("byte").run([&] {
-        util::Xor(data, key);
+    const Obfuscation obfuscation{rng.rand64()};
+    assert(obfuscation);
+
+    size_t offset{0};
+    bench.batch(test_data.size()).unit("byte").run([&] {
+        obfuscation(test_data, offset++);
+        ankerl::nanobench::doNotOptimizeAway(test_data);
     });
 }
 

src/consensus/tx_check.cpp

@@ -38,22 +38,36 @@ bool CheckTransaction(const CTransaction& tx, TxValidationState& state)
     // of a tx as spent, it does not check if the tx has duplicate inputs.
     // Failure to run this check will result in either a crash or an inflation bug, depending on the implementation of
     // the underlying coins database.
-    std::set<COutPoint> vInOutPoints;
-    for (const auto& txin : tx.vin) {
-        if (!vInOutPoints.insert(txin.prevout).second)
+    if (tx.vin.size() == 1) {
+        if (tx.IsCoinBase()) {
+            if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) {
+                return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-cb-length");
+            }
+        }
+    } else if (tx.vin.size() == 2) {
+        if (tx.vin[0].prevout == tx.vin[1].prevout) {
             return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-inputs-duplicate");
-    }
+        }
+        if (tx.vin[0].prevout.IsNull() || tx.vin[1].prevout.IsNull()) {
+            return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-prevout-null");
+        }
+    } else {
+        std::vector<COutPoint> sortedPrevouts;
+        sortedPrevouts.reserve(tx.vin.size());
+        for (const auto& txin : tx.vin) {
+            sortedPrevouts.push_back(txin.prevout);
+        }
+        std::sort(sortedPrevouts.begin(), sortedPrevouts.end());
+        if (std::ranges::adjacent_find(sortedPrevouts) != sortedPrevouts.end()) {
+            return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-inputs-duplicate");
+        }
 
-    if (tx.IsCoinBase())
-    {
-        if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100)
-            return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-cb-length");
-    }
-    else
-    {
-        for (const auto& txin : tx.vin)
-            if (txin.prevout.IsNull())
+        for (const auto& in : sortedPrevouts) {
+            if (!in.hash.IsNull()) break; // invalid values can only be at the beginning
+            if (in.IsNull()) {
                 return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-prevout-null");
+            }
+        }
     }
 
     return true;

src/crypto/siphash.cpp

@@ -17,10 +17,10 @@
 
 CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
 {
-    v[0] = 0x736f6d6570736575ULL ^ k0;
-    v[1] = 0x646f72616e646f6dULL ^ k1;
-    v[2] = 0x6c7967656e657261ULL ^ k0;
-    v[3] = 0x7465646279746573ULL ^ k1;
+    v[0] = C0 ^ k0;
+    v[1] = C1 ^ k1;
+    v[2] = C2 ^ k0;
+    v[3] = C3 ^ k1;
     count = 0;
     tmp = 0;
 }
@@ -97,10 +97,10 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val)
     /* Specialized implementation for efficiency */
     uint64_t d = val.GetUint64(0);
 
-    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
-    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
-    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
-    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
+    uint64_t v0 = CSipHasher::C0 ^ k0;
+    uint64_t v1 = CSipHasher::C1 ^ k1;
+    uint64_t v2 = CSipHasher::C2 ^ k0;
+    uint64_t v3 = CSipHasher::C3 ^ k1 ^ d;
 
     SIPROUND;
     SIPROUND;
@@ -132,16 +132,12 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val)
     return v0 ^ v1 ^ v2 ^ v3;
 }
 
-uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra)
+/* Specialized implementation for efficiency */
+uint64_t Uint256ExtraSipHasher::operator()(const uint256& val, uint32_t extra) const noexcept
 {
-    /* Specialized implementation for efficiency */
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
     uint64_t d = val.GetUint64(0);
-
-    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
-    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
-    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
-    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
-
+    v3 ^= d;
     SIPROUND;
     SIPROUND;
     v0 ^= d;

src/crypto/siphash.h

@@ -19,6 +19,11 @@ private:
     uint8_t count; // Only the low 8 bits of the input size matter.
 
 public:
+    static constexpr uint64_t C0{0x736f6d6570736575ULL};
+    static constexpr uint64_t C1{0x646f72616e646f6dULL};
+    static constexpr uint64_t C2{0x6c7967656e657261ULL};
+    static constexpr uint64_t C3{0x7465646279746573ULL};
+
     /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
     CSipHasher(uint64_t k0, uint64_t k1);
     /** Hash a 64-bit integer worth of data
@@ -43,6 +48,19 @@ public:
  *      .Finalize()
  */
 uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
-uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra);
+
+class Uint256ExtraSipHasher {
+    uint64_t v[4];
+
+public:
+    Uint256ExtraSipHasher(const uint64_t k0, const uint64_t k1) noexcept {
+        v[0] = CSipHasher::C0 ^ k0;
+        v[1] = CSipHasher::C1 ^ k1;
+        v[2] = CSipHasher::C2 ^ k0;
+        v[3] = CSipHasher::C3 ^ k1;
+    }
+
+    uint64_t operator()(const uint256& val, uint32_t extra) const noexcept;
+};
 
 #endif // BITCOIN_CRYPTO_SIPHASH_H

src/dbwrapper.cpp

@@ -171,7 +171,7 @@ void CDBBatch::Clear()
 void CDBBatch::WriteImpl(Span<const std::byte> key, DataStream& ssValue)
 {
     leveldb::Slice slKey(CharCast(key.data()), key.size());
-    ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
+    dbwrapper_private::GetObfuscation(parent)(ssValue);
     leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
     m_impl_batch->batch.Put(slKey, slValue);
     // LevelDB serializes writes as:
@@ -220,7 +220,11 @@ struct LevelDBContext {
 };
 
 CDBWrapper::CDBWrapper(const DBParams& params)
-    : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
+    : m_db_context{std::make_unique<LevelDBContext>()},
+      m_name{fs::PathToString(params.path.stem())},
+      m_obfuscation{0},
+      m_path{params.path},
+      m_is_memory{params.memory_only}
 {
     DBContext().penv = nullptr;
     DBContext().readoptions.verify_checksums = true;
@@ -255,24 +259,23 @@ CDBWrapper::CDBWrapper(const DBParams& params)
         LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
     }
 
-    // The base-case obfuscation key, which is a noop.
-    obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
-
-    bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
-
-    if (!key_exists && params.obfuscate && IsEmpty()) {
-        // Initialize non-degenerate obfuscation if it won't upset
-        // existing, non-obfuscated data.
-        std::vector<unsigned char> new_key = CreateObfuscateKey();
+    m_obfuscation = 0; // Needed for unobfuscated Read
+    std::vector<unsigned char> obfuscate_key_vector(Obfuscation::SIZE_BYTES, '\000');
+    const bool key_missing = !Read(OBFUSCATE_KEY_KEY, obfuscate_key_vector);
+    if (key_missing && params.obfuscate && IsEmpty()) {
+        // Initialize non-degenerate obfuscation if it won't upset existing, non-obfuscated data.
+        std::vector<uint8_t> new_key(Obfuscation::SIZE_BYTES);
+        GetRandBytes(new_key);
 
         // Write `new_key` so we don't obfuscate the key with itself
         Write(OBFUSCATE_KEY_KEY, new_key);
-        obfuscate_key = new_key;
+        obfuscate_key_vector = new_key;
 
-        LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
+        LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
     }
-
-    LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
+    LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
+    m_obfuscation = obfuscate_key_vector;
+    obfuscate_key_vector.clear();
 }
 
 CDBWrapper::~CDBWrapper()
@@ -323,19 +326,6 @@ size_t CDBWrapper::DynamicMemoryUsage() const
 // past the null-terminator.
 const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
 
-const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
-
-/**
- * Returns a string (consisting of 8 random bytes) suitable for use as an
- * obfuscating XOR key.
- */
-std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
-{
-    std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
-    GetRandBytes(ret);
-    return ret;
-}
-
 std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
 {
     leveldb::Slice slKey(CharCast(key.data()), key.size());
@@ -418,10 +408,5 @@ void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
 void CDBIterator::Next() { m_impl_iter->iter->Next(); }
 
 namespace dbwrapper_private {
-
-const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
-{
-    return w.obfuscate_key;
-}
-
+Obfuscation GetObfuscation(const CDBWrapper& w) { return w.m_obfuscation; }
 } // namespace dbwrapper_private

src/dbwrapper.h

@@ -63,8 +63,7 @@ namespace dbwrapper_private {
  * Database obfuscation should be considered an implementation detail of the
  * specific database.
  */
-const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
-
+Obfuscation GetObfuscation(const CDBWrapper&);
 }; // namespace dbwrapper_private
 
 bool DestroyDB(const std::string& path_str);
@@ -168,7 +167,7 @@ public:
     template<typename V> bool GetValue(V& value) {
         try {
             DataStream ssValue{GetValueImpl()};
-            ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
+            dbwrapper_private::GetObfuscation(parent)(ssValue);
             ssValue >> value;
         } catch (const std::exception&) {
             return false;
@@ -181,7 +180,7 @@ struct LevelDBContext;
 
 class CDBWrapper
 {
-    friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
+    friend Obfuscation dbwrapper_private::GetObfuscation(const CDBWrapper&);
 private:
     //! holds all leveldb-specific fields of this class
     std::unique_ptr<LevelDBContext> m_db_context;
@@ -190,16 +189,11 @@ private:
     std::string m_name;
 
     //! a key used for optional XOR-obfuscation of the database
-    std::vector<unsigned char> obfuscate_key;
+    Obfuscation m_obfuscation;
 
     //! the key under which the obfuscation key is stored
     static const std::string OBFUSCATE_KEY_KEY;
 
-    //! the length of the obfuscate key in number of bytes
-    static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
-
-    std::vector<unsigned char> CreateObfuscateKey() const;
-
     //! path to filesystem storage
     const fs::path m_path;
 
@@ -230,7 +224,7 @@ public:
         }
         try {
             DataStream ssValue{MakeByteSpan(*strValue)};
-            ssValue.Xor(obfuscate_key);
+            m_obfuscation(ssValue);
             ssValue >> value;
         } catch (const std::exception&) {
             return false;

src/hash.h

@@ -107,6 +107,10 @@ public:
     {
         ctx.Write(UCharCast(src.data()), src.size());
     }
+    void write(std::byte src)
+    {
+        ctx.Write(UCharCast(&src), 1);
+    }
 
     /** Compute the double-SHA256 hash of all data written to this object.
      *
@@ -194,6 +198,11 @@ public:
         m_source.write(src);
         HashWriter::write(src);
     }
+    void write(std::byte src)
+    {
+        m_source.write(src);
+        HashWriter::write(src);
+    }
 
     template <typename T>
     HashedSourceWriter& operator<<(const T& obj)

src/node/blockstorage.cpp

@@ -657,33 +657,48 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
 
 bool BlockManager::ReadBlockUndo(CBlockUndo& blockundo, const CBlockIndex& index) const
 {
-    const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
+    FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
+    if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) {
+        LogError("%s: OpenUndoFile failed for %s while reading", __func__, pos.ToString());
+        return false;
+    }
+    uint32_t undo_size;
+    pos.nPos -= sizeof undo_size;
 
     // Open history file to read
     AutoFile filein{OpenUndoFile(pos, true)};
     if (filein.IsNull()) {
-        LogError("OpenUndoFile failed for %s", pos.ToString());
+        LogError("OpenUndoFile failed for %s while reading", pos.ToString());
         return false;
     }
 
-    // Read block
-    uint256 hashChecksum;
-    HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
     try {
+        // Read block
+        filein >> undo_size;
+        if (undo_size > MAX_SIZE) {
+            LogError("Refusing to read undo data of size: %d", undo_size);
+            return false;
+        }
+
+        std::vector<uint8_t> mem(undo_size);
+        filein >> Span{mem};
+
+        SpanReader reader{mem};
+        HashVerifier verifier{reader}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
         verifier << index.pprev->GetBlockHash();
         verifier >> blockundo;
+
+        uint256 hashChecksum;
         filein >> hashChecksum;
+        if (hashChecksum != verifier.GetHash()) {
+            LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
+            return false;
+        }
     } catch (const std::exception& e) {
         LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
         return false;
     }
 
-    // Verify checksum
-    if (hashChecksum != verifier.GetHash()) {
-        LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
-        return false;
-    }
-
     return true;
 }
 
@@ -774,13 +789,13 @@ void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
 
 AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
 {
-    return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key};
+    return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation};
 }
 
 /** Open an undo file (rev?????.dat) */
 AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
 {
-    return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key};
+    return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation};
 }
 
 fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
@@ -933,27 +948,27 @@ bool BlockManager::WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationSt
         FlatFilePos pos;
         const unsigned int blockundo_size{static_cast<unsigned int>(GetSerializeSize(blockundo))};
         if (!FindUndoPos(state, block.nFile, pos, blockundo_size + UNDO_DATA_DISK_OVERHEAD)) {
-            LogError("FindUndoPos failed");
+            LogError("FindUndoPos failed for %s while writing", pos.ToString());
             return false;
         }
         // Open history file to append
         AutoFile fileout{OpenUndoFile(pos)};
         if (fileout.IsNull()) {
-            LogError("OpenUndoFile failed");
+            LogError("OpenUndoFile failed for %s while writing", pos.ToString());
             return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
         }
 
         // Write index header
-        fileout << GetParams().MessageStart() << blockundo_size;
-        // Write undo data
+        fileout.write_large(DataStream(BLOCK_SERIALIZATION_HEADER_SIZE) << GetParams().MessageStart() << blockundo_size);
         pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
-        fileout << blockundo;
+        {
+            // Calculate checksum
+            HashWriter hasher{};
+            hasher << block.pprev->GetBlockHash() << blockundo;
 
-        // Calculate & write checksum
-        HashWriter hasher{};
-        hasher << block.pprev->GetBlockHash();
-        hasher << blockundo;
-        fileout << hasher.GetHash();
+            // Write undo data & checksum
+            fileout.write_large(DataStream(blockundo_size + sizeof(uint256)) << blockundo << hasher.GetHash());
+        }
 
         // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
         // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
@@ -981,34 +996,49 @@ bool BlockManager::WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationSt
     return true;
 }
 
-bool BlockManager::ReadBlock(CBlock& block, const FlatFilePos& pos) const
+bool BlockManager::ReadBlock(CBlock& block, FlatFilePos pos) const
 {
     block.SetNull();
 
+    if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) {
+        LogError("%s: OpenBlockFile failed for %s", __func__, pos.ToString());
+        return false;
+    }
+    uint32_t blk_size;
+    pos.nPos -= sizeof blk_size;
+
     // Open history file to read
     AutoFile filein{OpenBlockFile(pos, true)};
     if (filein.IsNull()) {
-        LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
+        LogError("%s: OpenBlockFile failed for %s", __func__, pos.ToString());
         return false;
     }
 
-    // Read block
     try {
-        filein >> TX_WITH_WITNESS(block);
+        // Read block
+        filein >> blk_size;
+        if (blk_size > MAX_SIZE) {
+            LogError("Refusing to read block of size: %s", blk_size);
+            return false;
+        }
+
+        std::vector<uint8_t> mem(blk_size);
+        filein >> Span{mem};
+        SpanReader(mem) >> TX_WITH_WITNESS(block);
     } catch (const std::exception& e) {
-        LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
+        LogError("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString());
         return false;
     }
 
     // Check the header
     if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) {
-        LogError("%s: Errors in block header at %s\n", __func__, pos.ToString());
+        LogError("%s: Errors in block header at %s", __func__, pos.ToString());
         return false;
     }
 
     // Signet only: check block solution
     if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
-        LogError("%s: Errors in block solution at %s\n", __func__, pos.ToString());
+        LogError("%s: Errors in block solution at %s", __func__, pos.ToString());
         return false;
     }
 
@@ -1023,7 +1053,7 @@ bool BlockManager::ReadBlock(CBlock& block, const CBlockIndex& index) const
         return false;
     }
     if (block.GetHash() != index.GetBlockHash()) {
-        LogError("%s: GetHash() doesn't match index for %s at %s\n", __func__, index.ToString(), block_pos.ToString());
+        LogError("%s: GetHash() doesn't match index for %s at %s", __func__, index.ToString(), block_pos.ToString());
         return false;
     }
     return true;
@@ -1035,13 +1065,13 @@ bool BlockManager::ReadRawBlock(std::vector<uint8_t>& block, const FlatFilePos&
     // If nPos is less than 8 the pos is null and we don't have the block data
     // Return early to prevent undefined behavior of unsigned int underflow
     if (hpos.nPos < 8) {
-        LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
+        LogError("%s: OpenBlockFile failed for %s", __func__, pos.ToString());
         return false;
     }
     hpos.nPos -= 8; // Seek back 8 bytes for meta header
     AutoFile filein{OpenBlockFile(hpos, true)};
     if (filein.IsNull()) {
-        LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
+        LogError("%s: OpenBlockFile failed for %s", __func__, pos.ToString());
         return false;
     }
 
@@ -1052,14 +1082,14 @@ bool BlockManager::ReadRawBlock(std::vector<uint8_t>& block, const FlatFilePos&
         filein >> blk_start >> blk_size;
 
         if (blk_start != GetParams().MessageStart()) {
-            LogError("%s: Block magic mismatch for %s: %s versus expected %s\n", __func__, pos.ToString(),
+            LogError("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(),
                          HexStr(blk_start),
                          HexStr(GetParams().MessageStart()));
             return false;
         }
 
         if (blk_size > MAX_SIZE) {
-            LogError("%s: Block data is larger than maximum deserialization size for %s: %s versus %s\n", __func__, pos.ToString(),
+            LogError("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(),
                          blk_size, MAX_SIZE);
             return false;
         }
@@ -1067,7 +1097,7 @@ bool BlockManager::ReadRawBlock(std::vector<uint8_t>& block, const FlatFilePos&
         block.resize(blk_size); // Zeroing of memory is intentional here
         filein.read(MakeWritableByteSpan(block));
     } catch (const std::exception& e) {
-        LogError("%s: Read from block file failed: %s for %s\n", __func__, e.what(), pos.ToString());
+        LogError("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString());
         return false;
     }
 
@@ -1079,25 +1109,26 @@ FlatFilePos BlockManager::WriteBlock(const CBlock& block, int nHeight)
     const unsigned int block_size{static_cast<unsigned int>(GetSerializeSize(TX_WITH_WITNESS(block)))};
     FlatFilePos pos{FindNextBlockPos(block_size + BLOCK_SERIALIZATION_HEADER_SIZE, nHeight, block.GetBlockTime())};
     if (pos.IsNull()) {
-        LogError("FindNextBlockPos failed");
+        LogError("FindNextBlockPos failed for %s while writing", pos.ToString());
         return FlatFilePos();
     }
     AutoFile fileout{OpenBlockFile(pos)};
     if (fileout.IsNull()) {
-        LogError("OpenBlockFile failed");
+        LogError("OpenBlockFile failed for %s while writing", pos.ToString());
         m_opts.notifications.fatalError(_("Failed to write block."));
         return FlatFilePos();
     }
 
     // Write index header
-    fileout << GetParams().MessageStart() << block_size;
-    // Write block
+    fileout.write_large(DataStream(BLOCK_SERIALIZATION_HEADER_SIZE) << GetParams().MessageStart() << block_size);
     pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
-    fileout << TX_WITH_WITNESS(block);
+    // Write block
+    fileout.write_large(DataStream(block_size) << TX_WITH_WITNESS(block));
+
     return pos;
 }
 
-static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
+static Obfuscation InitBlocksdirXorKey(const BlockManager::Options& opts)
 {
     // Bytes are serialized without length indicator, so this is also the exact
     // size of the XOR-key file.
@@ -1146,12 +1177,12 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
         };
     }
     LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
-    return std::vector<std::byte>{xor_key.begin(), xor_key.end()};
+    return Obfuscation{xor_key};
 }
 
 BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
     : m_prune_mode{opts.prune_target > 0},
-      m_xor_key{InitBlocksdirXorKey(opts)},
+      m_obfuscation{InitBlocksdirXorKey(opts)},
       m_opts{std::move(opts)},
       m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
       m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},

src/node/blockstorage.h

@@ -235,7 +235,7 @@ private:
 
     const bool m_prune_mode;
 
-    const std::vector<std::byte> m_xor_key;
+    const Obfuscation m_obfuscation;
 
     /** Dirty block index entries. */
     std::set<CBlockIndex*> m_dirty_blockindex;
@@ -411,7 +411,7 @@ public:
     void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
 
     /** Functions for disk access for blocks */
-    bool ReadBlock(CBlock& block, const FlatFilePos& pos) const;
+    bool ReadBlock(CBlock& block, FlatFilePos pos) const;
     bool ReadBlock(CBlock& block, const CBlockIndex& index) const;
     bool ReadRawBlock(std::vector<uint8_t>& block, const FlatFilePos& pos) const;
 

src/node/mempool_persist.cpp

@@ -58,15 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
     try {
         uint64_t version;
         file >> version;
-        std::vector<std::byte> xor_key;
         if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
-            // Leave XOR-key empty
+            file.SetObfuscation(0);
         } else if (version == MEMPOOL_DUMP_VERSION) {
-            file >> xor_key;
+            Obfuscation obfuscation{0};
+            file >> obfuscation;
+            file.SetObfuscation(obfuscation);
         } else {
             return false;
         }
-        file.SetXor(xor_key);
         uint64_t total_txns_to_load;
         file >> total_txns_to_load;
         uint64_t txns_tried = 0;
@@ -177,12 +177,13 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock
         const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION};
         file << version;
 
-        std::vector<std::byte> xor_key(8);
         if (!pool.m_opts.persist_v1_dat) {
-            FastRandomContext{}.fillrand(xor_key);
-            file << xor_key;
+            const Obfuscation obfuscation{FastRandomContext{}.rand64()};
+            file << obfuscation;
+            file.SetObfuscation(obfuscation);
+        } else {
+            file.SetObfuscation(0);
         }
-        file.SetXor(xor_key);
 
         uint64_t mempool_transactions_to_write(vinfo.size());
         file << mempool_transactions_to_write;

src/obfuscation.h

@@ -0,0 +1,86 @@
+// Copyright (c) 2009-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_OBFUSCATION_H
+#define BITCOIN_OBFUSCATION_H
+
+#include <array>
+#include <cassert>
+#include <cstdint>
+#include <random>
+#include <span.h>
+#include <util/check.h>
+#include <cstring>
+#include <climits>
+#include <serialize.h>
+
+class Obfuscation
+{
+public:
+    static constexpr size_t SIZE_BYTES{sizeof(uint64_t)};
+
+private:
+    std::array<uint64_t, SIZE_BYTES> rotations; // Cached key rotations
+    void SetRotations(const uint64_t key)
+    {
+        for (size_t i{0}; i < SIZE_BYTES; ++i)
+        {
+            size_t key_rotation_bits{CHAR_BIT * i};
+            if constexpr (std::endian::native == std::endian::big) key_rotation_bits *= -1;
+            rotations[i] = std::rotr(key, key_rotation_bits);
+        }
+    }
+
+    static uint64_t ToUint64(const Span<const std::byte> key_span)
+    {
+        uint64_t key{};
+        std::memcpy(&key, key_span.data(), SIZE_BYTES);
+        return key;
+    }
+
+    static void Xor(Span<std::byte> write, const uint64_t key, const size_t size)
+    {
+        assert(size <= write.size());
+        uint64_t raw{};
+        std::memcpy(&raw, write.data(), size);
+        raw ^= key;
+        std::memcpy(write.data(), &raw, size);
+    }
+
+public:
+    Obfuscation(const uint64_t key) { SetRotations(key); }
+    Obfuscation(const Span<const std::byte> key_span) : Obfuscation(ToUint64(key_span)) {}
+    Obfuscation(const std::array<const std::byte, SIZE_BYTES>& key_arr) : Obfuscation(ToUint64(key_arr)) {}
+    Obfuscation(const std::vector<uint8_t>& key_vec) : Obfuscation(MakeByteSpan(key_vec)) {}
+
+    uint64_t Key() const { return rotations[0]; }
+    operator bool() const { return Key() != 0; }
+    void operator()(Span<std::byte> write, const size_t key_offset_bytes = 0) const
+    {
+        if (!*this) return;
+        const uint64_t rot_key{rotations[key_offset_bytes % SIZE_BYTES]}; // Continue obfuscation from where we left off
+        for (; write.size() >= SIZE_BYTES; write = write.subspan(SIZE_BYTES)) { // Process multiple bytes at a time
+            Xor(write, rot_key, SIZE_BYTES);
+        }
+        Xor(write, rot_key, write.size());
+    }
+
+    template <typename Stream>
+    void Serialize(Stream& s) const
+    {
+        std::vector<std::byte> bytes(SIZE_BYTES);
+        std::memcpy(bytes.data(), &rotations[0], SIZE_BYTES);
+        s << bytes;
+    }
+
+    template <typename Stream>
+    void Unserialize(Stream& s)
+    {
+        std::vector<std::byte> bytes(SIZE_BYTES);
+        s >> bytes;
+        SetRotations(ToUint64(bytes));
+    }
+};
+
+#endif // BITCOIN_OBFUSCATION_H

src/serialize.h

@@ -48,23 +48,28 @@ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
 struct deserialize_type {};
 constexpr deserialize_type deserialize {};
 
+class SizeComputer;
+
+//! Check if type contains a stream by seeing if it has a GetStream() method.
+template<typename T>
+concept ContainsStream = requires(T t) { t.GetStream(); };
+
+template<typename T>
+concept ContainsSizeComputer = ContainsStream<T> &&
+    std::is_same_v<std::remove_reference_t<decltype(std::declval<T>().GetStream())>, SizeComputer>;
+
 /*
  * Lowest-level serialization and conversion.
  */
 template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
 {
-    s.write(AsBytes(Span{&obj, 1}));
+    s.write(std::byte{obj});
 }
 template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
 {
     obj = htole16_internal(obj);
     s.write(AsBytes(Span{&obj, 1}));
 }
-template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
-{
-    obj = htobe16_internal(obj);
-    s.write(AsBytes(Span{&obj, 1}));
-}
 template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
 {
     obj = htole32_internal(obj);
@@ -92,12 +97,6 @@ template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
     s.read(AsWritableBytes(Span{&obj, 1}));
     return le16toh_internal(obj);
 }
-template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
-{
-    uint16_t obj;
-    s.read(AsWritableBytes(Span{&obj, 1}));
-    return be16toh_internal(obj);
-}
 template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
 {
     uint32_t obj;
@@ -118,8 +117,6 @@ template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
 }
 
 
-class SizeComputer;
-
 /**
  * Convert any argument to a reference to X, maintaining constness.
  *
@@ -252,38 +249,49 @@ const Out& AsBase(const In& x)
 template<class T>
 concept CharNotInt8 = std::same_as<T, char> && !std::same_as<T, int8_t>;
 
+template <typename T>
+concept ByteOrIntegral = std::is_same_v<T, std::byte> ||
+    (std::is_integral_v<T> && !std::is_same_v<T, char>);
+
 template <typename Stream, CharNotInt8 V> void Serialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
-template <typename Stream> void Serialize(Stream& s, std::byte a) { ser_writedata8(s, uint8_t(a)); }
-template<typename Stream> inline void Serialize(Stream& s, int8_t a  ) { ser_writedata8(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
-template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
+template <typename Stream, ByteOrIntegral T> void Serialize(Stream& s, T a)
+{
+    if constexpr (ContainsSizeComputer<Stream>) {
+        s.GetStream().seek(sizeof(T));
+    } else if constexpr (sizeof(T) == 1) {
+        ser_writedata8(s, static_cast<uint8_t>(a));   // (u)int8_t or std::byte or bool
+    } else if constexpr (sizeof(T) == 2) {
+        ser_writedata16(s, static_cast<uint16_t>(a)); // (u)int16_t
+    } else if constexpr (sizeof(T) == 4) {
+        ser_writedata32(s, static_cast<uint32_t>(a)); // (u)int32_t
+    } else {
+        static_assert(sizeof(T) == 8);
+        ser_writedata64(s, static_cast<uint64_t>(a)); // (u)int64_t
+    }
+}
 template <typename Stream, BasicByte B, int N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); }
 template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); }
 template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, std::span<B, N> span) { s.write(std::as_bytes(span)); }
 template <typename Stream, BasicByte B> void Serialize(Stream& s, Span<B> span) { s.write(AsBytes(span)); }
 
 template <typename Stream, CharNotInt8 V> void Unserialize(Stream&, V) = delete; // char serialization forbidden. Use uint8_t or int8_t
-template <typename Stream> void Unserialize(Stream& s, std::byte& a) { a = std::byte{ser_readdata8(s)}; }
-template<typename Stream> inline void Unserialize(Stream& s, int8_t& a  ) { a = ser_readdata8(s); }
-template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
-template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
-template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
-template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
-template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
-template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
-template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
+template <typename Stream, ByteOrIntegral T> void Unserialize(Stream& s, T& a)
+{
+    if constexpr (sizeof(T) == 1) {
+        a = static_cast<T>(ser_readdata8(s));  // (u)int8_t or std::byte or bool
+    } else if constexpr (sizeof(T) == 2) {
+        a = static_cast<T>(ser_readdata16(s)); // (u)int16_t
+    } else if constexpr (sizeof(T) == 4) {
+        a = static_cast<T>(ser_readdata32(s)); // (u)int32_t
+    } else {
+        static_assert(sizeof(T) == 8);
+        a = static_cast<T>(ser_readdata64(s)); // (u)int64_t
+    }
+}
 template <typename Stream, BasicByte B, int N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); }
 template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); }
 template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::span<B, N> span) { s.read(std::as_writable_bytes(span)); }
 template <typename Stream, BasicByte B> void Unserialize(Stream& s, Span<B> span) { s.read(AsWritableBytes(span)); }
-
-template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); }
-template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; }
 // clang-format on
 
 
@@ -302,12 +310,14 @@ constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
     else                         return sizeof(unsigned char) + sizeof(uint64_t);
 }
 
-inline void WriteCompactSize(SizeComputer& os, uint64_t nSize);
-
 template<typename Stream>
 void WriteCompactSize(Stream& os, uint64_t nSize)
 {
-    if (nSize < 253)
+    if constexpr (ContainsSizeComputer<Stream>)
+    {
+        os.GetStream().seek(GetSizeOfCompactSize(nSize));
+    }
+    else if (nSize < 253)
     {
         ser_writedata8(os, nSize);
     }
@@ -414,7 +424,7 @@ struct CheckVarIntMode {
 };
 
 template<VarIntMode Mode, typename I>
-inline unsigned int GetSizeOfVarInt(I n)
+constexpr unsigned int GetSizeOfVarInt(I n)
 {
     CheckVarIntMode<Mode, I>();
     int nRet = 0;
@@ -427,25 +437,26 @@ inline unsigned int GetSizeOfVarInt(I n)
     return nRet;
 }
 
-template<typename I>
-inline void WriteVarInt(SizeComputer& os, I n);
-
 template<typename Stream, VarIntMode Mode, typename I>
 void WriteVarInt(Stream& os, I n)
 {
-    CheckVarIntMode<Mode, I>();
-    unsigned char tmp[(sizeof(n)*8+6)/7];
-    int len=0;
-    while(true) {
-        tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
-        if (n <= 0x7F)
-            break;
-        n = (n >> 7) - 1;
-        len++;
+    if constexpr (ContainsSizeComputer<Stream>) {
+        os.GetStream().seek(GetSizeOfVarInt<Mode, I>(n));
+    } else {
+        CheckVarIntMode<Mode, I>();
+        unsigned char tmp[(sizeof(n)*8+6)/7];
+        int len=0;
+        while(true) {
+            tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
+            if (n <= 0x7F)
+                break;
+            n = (n >> 7) - 1;
+            len++;
+        }
+        do {
+            ser_writedata8(os, tmp[len]);
+        } while(len--);
     }
-    do {
-        ser_writedata8(os, tmp[len]);
-    } while(len--);
 }
 
 template<typename Stream, VarIntMode Mode, typename I>
@@ -489,7 +500,7 @@ public:
  * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside
  * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object).
  *
- * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is
+ * This works by constructing a Wrapper<Formatter, T&>-wrapped version of object, where T is
  * const during serialization, and non-const during deserialization, which maintains const
  * correctness.
  */
@@ -534,7 +545,9 @@ struct CustomUintFormatter
     template <typename Stream, typename I> void Ser(Stream& s, I v)
     {
         if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
-        if (BigEndian) {
+        if constexpr (ContainsSizeComputer<Stream>) {
+            s.GetStream().seek(Bytes);
+        } else if (BigEndian) {
             uint64_t raw = htobe64_internal(v);
             s.write(AsBytes(Span{&raw, 1}).last(Bytes));
         } else {
@@ -1065,10 +1078,17 @@ protected:
 public:
     SizeComputer() = default;
 
+    SizeComputer& GetStream() { return *this; }
+    const SizeComputer& GetStream() const { return *this; };
+
     void write(Span<const std::byte> src)
     {
         this->nSize += src.size();
     }
+    void write(std::byte)
+    {
+        this->nSize += 1;
+    }
 
     /** Pretend _nSize bytes are written, without specifying them. */
     void seek(size_t _nSize)
@@ -1088,27 +1108,12 @@ public:
     }
 };
 
-template<typename I>
-inline void WriteVarInt(SizeComputer &s, I n)
-{
-    s.seek(GetSizeOfVarInt<I>(n));
-}
-
-inline void WriteCompactSize(SizeComputer &s, uint64_t nSize)
-{
-    s.seek(GetSizeOfCompactSize(nSize));
-}
-
 template <typename T>
 size_t GetSerializeSize(const T& t)
 {
     return (SizeComputer() << t).size();
 }
 
-//! Check if type contains a stream by seeing if has a GetStream() method.
-template<typename T>
-concept ContainsStream = requires(T t) { t.GetStream(); };
-
 /** Wrapper that overrides the GetParams() function of a stream. */
 template <typename SubStream, typename Params>
 class ParamsStream
@@ -1133,6 +1138,7 @@ public:
     template <typename U> ParamsStream& operator<<(const U& obj) { ::Serialize(*this, obj); return *this; }
     template <typename U> ParamsStream& operator>>(U&& obj) { ::Unserialize(*this, obj); return *this; }
     void write(Span<const std::byte> src) { GetStream().write(src); }
+    void write(std::byte src) { GetStream().write(src); }
     void read(Span<std::byte> dst) { GetStream().read(dst); }
     void ignore(size_t num) { GetStream().ignore(num); }
     bool eof() const { return GetStream().eof(); }

src/streams.cpp

@@ -9,8 +9,7 @@
 
 #include <array>
 
-AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
-    : m_file{file}, m_xor{std::move(data_xor)}
+AutoFile::AutoFile(std::FILE* file, const Obfuscation& obfuscation) : m_file{file}, m_obfuscation{obfuscation}
 {
     if (!IsNull()) {
         auto pos{std::ftell(m_file)};
@@ -21,12 +20,12 @@ AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
 std::size_t AutoFile::detail_fread(Span<std::byte> dst)
 {
     if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr");
-    size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
-    if (!m_xor.empty()) {
-        if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown");
-        util::Xor(dst.subspan(0, ret), m_xor, *m_position);
+    const size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
+    if (m_obfuscation) {
+        if (!m_position) throw std::ios_base::failure("AutoFile::read: position unknown");
+        m_obfuscation(dst, *m_position);
     }
-    if (m_position.has_value()) *m_position += ret;
+    if (m_position) *m_position += ret;
     return ret;
 }
 
@@ -81,7 +80,7 @@ void AutoFile::ignore(size_t nSize)
 void AutoFile::write(Span<const std::byte> src)
 {
     if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr");
-    if (m_xor.empty()) {
+    if (!m_obfuscation) {
         if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
             throw std::ios_base::failure("AutoFile::write: write failed");
         }
@@ -91,8 +90,8 @@ void AutoFile::write(Span<const std::byte> src)
         std::array<std::byte, 4096> buf;
         while (src.size() > 0) {
             auto buf_now{Span{buf}.first(std::min<size_t>(src.size(), buf.size()))};
-            std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin());
-            util::Xor(buf_now, m_xor, *m_position);
+            std::copy_n(src.begin(), buf_now.size(), buf_now.begin());
+            m_obfuscation(buf_now, *m_position);
             if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) {
                 throw std::ios_base::failure{"XorFile::write: failed"};
             }
@@ -102,6 +101,34 @@ void AutoFile::write(Span<const std::byte> src)
     }
 }
 
+void AutoFile::write_large(Span<std::byte> src)
+{
+    if (!m_file) throw std::ios_base::failure("AutoFile::write_large: file handle is nullptr");
+    m_obfuscation(src, *m_position); // obfuscate in-place
+    if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
+        throw std::ios_base::failure("AutoFile::write_large: write failed");
+    }
+    if (m_position) *m_position += src.size();
+}
+
+void AutoFile::write(std::byte val)
+{
+    if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr");
+    if (!m_obfuscation) {
+        if (fwrite(&val, 1, 1, m_file) != 1) {
+            throw std::ios_base::failure("AutoFile::write: write failed");
+        }
+        if (m_position.has_value()) *m_position += 1;
+    } else {
+        if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::write: position unknown");
+        auto src{Span{&val, 1}};
+        m_obfuscation(src, *m_position);
+        if (fwrite(src.data(), 1, 1, m_file) != 1) {
+            throw std::ios_base::failure{"XorFile::write: failed"};
+        }
+        *m_position += 1;
+    }
+}
 bool AutoFile::Commit()
 {
     return ::FileCommit(m_file);

src/streams.h

@@ -6,6 +6,7 @@
 #ifndef BITCOIN_STREAMS_H
 #define BITCOIN_STREAMS_H
 
+#include <obfuscation.h>
 #include <serialize.h>
 #include <span.h>
 #include <support/allocators/zeroafterfree.h>
@@ -21,30 +22,8 @@
 #include <stdint.h>
 #include <string.h>
 #include <string>
-#include <utility>
 #include <vector>
 
-namespace util {
-inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_offset = 0)
-{
-    if (key.size() == 0) {
-        return;
-    }
-    key_offset %= key.size();
-
-    for (size_t i = 0, j = key_offset; i != write.size(); i++) {
-        write[i] ^= key[j++];
-
-        // This potentially acts on very many bytes of data, so it's
-        // important that we calculate `j`, i.e. the `key` index in this
-        // way instead of doing a %, which would effectively be a division
-        // for each byte Xor'd -- much slower than need be.
-        if (j == key.size())
-            j = 0;
-    }
-}
-} // namespace util
-
 /* Minimal stream for overwriting and/or appending to an existing byte vector
  *
  * The referenced vector will grow as necessary
@@ -83,6 +62,16 @@ public:
         }
         nPos += src.size();
     }
+    void write(std::byte val)
+    {
+        assert(nPos <= vchData.size());
+        if (nPos < vchData.size()) {
+            vchData[nPos] = static_cast<unsigned char>(val);
+        } else {
+            vchData.push_back(static_cast<unsigned char>(val));
+        }
+        nPos += 1;
+    }
     template <typename T>
     VectorWriter& operator<<(const T& obj)
     {
@@ -162,6 +151,7 @@ public:
     typedef vector_type::reverse_iterator reverse_iterator;
 
     explicit DataStream() = default;
+    explicit DataStream(size_type n) { reserve(n); }
     explicit DataStream(Span<const uint8_t> sp) : DataStream{AsBytes(sp)} {}
     explicit DataStream(Span<const value_type> sp) : vch(sp.data(), sp.data() + sp.size()) {}
 
@@ -253,6 +243,11 @@ public:
         // Write to the end of the buffer
         vch.insert(vch.end(), src.begin(), src.end());
     }
+    void write(value_type val)
+    {
+        // Push single value to the end of the buffer
+        vch.push_back(val);
+    }
 
     template<typename T>
     DataStream& operator<<(const T& obj)
@@ -261,21 +256,16 @@ public:
         return (*this);
     }
 
-    template<typename T>
+    template <typename T>
     DataStream& operator>>(T&& obj)
     {
         ::Unserialize(*this, obj);
         return (*this);
     }
 
-    /**
-     * XOR the contents of this stream with a certain key.
-     *
-     * @param[in] key    The key used to XOR the data in this stream.
-     */
-    void Xor(const std::vector<unsigned char>& key)
+    void Obfuscate(const Obfuscation& obfuscation)
     {
-        util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key));
+        if (obfuscation) obfuscation(MakeWritableByteSpan(*this));
     }
 
     /** Compute total memory usage of this object (own memory + any dynamic memory). */
@@ -392,11 +382,11 @@ class AutoFile
 {
 protected:
     std::FILE* m_file;
-    std::vector<std::byte> m_xor;
+    Obfuscation m_obfuscation;
     std::optional<int64_t> m_position;
 
 public:
-    explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={});
+    explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = 0);
 
     ~AutoFile() { fclose(); }
 
@@ -428,7 +418,7 @@ public:
     bool IsNull() const { return m_file == nullptr; }
 
     /** Continue with a different XOR key */
-    void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; }
+    void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; }
 
     /** Implementation detail, only used internally. */
     std::size_t detail_fread(Span<std::byte> dst);
@@ -451,6 +441,8 @@ public:
     void read(Span<std::byte> dst);
     void ignore(size_t nSize);
     void write(Span<const std::byte> src);
+    void write_large(Span<std::byte> src);  // Note that src will be mutated
+    void write(std::byte src);
 
     template <typename T>
     AutoFile& operator<<(const T& obj)

src/test/dbwrapper_tests.cpp

@@ -14,16 +14,6 @@
 
 using util::ToString;
 
-// Test if a string consists entirely of null characters
-static bool is_null_key(const std::vector<unsigned char>& key) {
-    bool isnull = true;
-
-    for (unsigned int i = 0; i < key.size(); i++)
-        isnull &= (key[i] == '\x00');
-
-    return isnull;
-}
-
 BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup)
 
 BOOST_AUTO_TEST_CASE(dbwrapper)
@@ -37,7 +27,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
         uint256 res;
 
         // Ensure that we're doing real obfuscation when obfuscate=true
-        BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
+        BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
 
         BOOST_CHECK(dbw.Write(key, in));
         BOOST_CHECK(dbw.Read(key, res));
@@ -57,7 +47,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data)
         bool res_bool;
 
         // Ensure that we're doing real obfuscation when obfuscate=true
-        BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
+        BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
 
         //Simulate block raw data - "b + block hash"
         std::string key_block = "b" + m_rng.rand256().ToString();
@@ -232,7 +222,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
     BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
 
     BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
-    BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string
+    BOOST_CHECK(!dbwrapper_private::GetObfuscation(odbw));
 
     uint256 in2 = m_rng.rand256();
     uint256 res3;
@@ -269,7 +259,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex)
     // Check that the key/val we wrote with unobfuscated wrapper doesn't exist
     uint256 res2;
     BOOST_CHECK(!odbw.Read(key, res2));
-    BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw)));
+    BOOST_CHECK(dbwrapper_private::GetObfuscation(odbw));
 
     uint256 in2 = m_rng.rand256();
     uint256 res3;

src/test/fuzz/autofile.cpp

@@ -20,7 +20,7 @@ FUZZ_TARGET(autofile)
     FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider};
     AutoFile auto_file{
         fuzzed_file_provider.open(),
-        ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
+        fuzzed_data_provider.ConsumeIntegral<uint64_t>()
     };
     LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100)
     {

src/test/fuzz/buffered_file.cpp

@@ -22,7 +22,7 @@ FUZZ_TARGET(buffered_file)
     std::optional<BufferedFile> opt_buffered_file;
     AutoFile fuzzed_file{
         fuzzed_file_provider.open(),
-        ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
+        fuzzed_data_provider.ConsumeIntegral<uint64_t>()
     };
     try {
         auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096);

src/test/fuzz/integer.cpp

@@ -119,7 +119,7 @@ FUZZ_TARGET(integer, .init = initialize_integer)
     (void)MillisToTimeval(i64);
     (void)SighashToStr(uch);
     (void)SipHashUint256(u64, u64, u256);
-    (void)SipHashUint256Extra(u64, u64, u256, u32);
+    (void)Uint256ExtraSipHasher(u64, u64)(u256, u32);
     (void)ToLower(ch);
     (void)ToUpper(ch);
     {
@@ -236,10 +236,6 @@ FUZZ_TARGET(integer, .init = initialize_integer)
         const uint16_t deserialized_u16 = ser_readdata16(stream);
         assert(u16 == deserialized_u16 && stream.empty());
 
-        ser_writedata16be(stream, u16);
-        const uint16_t deserialized_u16be = ser_readdata16be(stream);
-        assert(u16 == deserialized_u16be && stream.empty());
-
         ser_writedata8(stream, u8);
         const uint8_t deserialized_u8 = ser_readdata8(stream);
         assert(u8 == deserialized_u8 && stream.empty());

src/test/hash_tests.cpp

@@ -130,21 +130,21 @@ BOOST_AUTO_TEST_CASE(siphash)
     ss << TX_WITH_WITNESS(tx);
     BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL);
 
-    // Check consistency between CSipHasher and SipHashUint256[Extra].
+    // Check consistency between CSipHasher and SipHashUint256 and Uint256ExtraSipHasher.
     FastRandomContext ctx;
     for (int i = 0; i < 16; ++i) {
+        uint64_t k0 = ctx.rand64();
         uint64_t k1 = ctx.rand64();
-        uint64_t k2 = ctx.rand64();
         uint256 x = m_rng.rand256();
         uint32_t n = ctx.rand32();
         uint8_t nb[4];
         WriteLE32(nb, n);
-        CSipHasher sip256(k1, k2);
+        CSipHasher sip256(k0, k1);
         sip256.Write(x);
         CSipHasher sip288 = sip256;
         sip288.Write(nb);
-        BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize());
-        BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize());
+        BOOST_CHECK_EQUAL(SipHashUint256(k0, k1, x), sip256.Finalize());
+        BOOST_CHECK_EQUAL(Uint256ExtraSipHasher(k0, k1)(x, n), sip288.Finalize());
     }
 }
 

src/test/streams_tests.cpp

@@ -14,16 +14,121 @@ using namespace std::string_literals;
 
 BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
 
+BOOST_AUTO_TEST_CASE(obfuscation_constructors)
+{
+    constexpr uint64_t test_key = 0x0123456789ABCDEF;
+
+    // Direct uint64_t constructor
+    const Obfuscation obf1{test_key};
+    BOOST_CHECK_EQUAL(obf1.Key(), test_key);
+
+    // Span constructor
+    std::array<std::byte, Obfuscation::SIZE_BYTES> key_bytes{};
+    std::memcpy(key_bytes.data(), &test_key, Obfuscation::SIZE_BYTES);
+    const Obfuscation obf2{Span{key_bytes}};
+    BOOST_CHECK_EQUAL(obf2.Key(), test_key);
+
+    // std::array<std:byte> constructor
+    const Obfuscation obf3{key_bytes};
+    BOOST_CHECK_EQUAL(obf3.Key(), test_key);
+
+    // std::vector<uint8_t> constructor
+    std::vector<uint8_t> uchar_key(Obfuscation::SIZE_BYTES);
+    std::memcpy(uchar_key.data(), &test_key, uchar_key.size());
+    const Obfuscation obf4{uchar_key};
+    BOOST_CHECK_EQUAL(obf4.Key(), test_key);
+}
+
+BOOST_AUTO_TEST_CASE(obfuscation_serialize)
+{
+    const Obfuscation original{0xDEADBEEF};
+
+    // Serialize
+    DataStream ds;
+    ds << original;
+
+    BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::SIZE_BYTES); // serialized as a vector
+
+    // Deserialize
+    Obfuscation recovered{0};
+    ds >> recovered;
+
+    BOOST_CHECK_EQUAL(recovered.Key(), original.Key());
+}
+
+BOOST_AUTO_TEST_CASE(obfuscation_empty)
+{
+    const Obfuscation null_obf{0};
+    BOOST_CHECK(!null_obf);
+}
+
+BOOST_AUTO_TEST_CASE(xor_bytes_reference)
+{
+    auto expected_xor{[](std::span<std::byte> write, const std::span<const std::byte> key, size_t key_offset) {
+        for (auto& b : write) {
+            b ^= key[key_offset++ % key.size()];
+        }
+    }};
+
+    FastRandomContext rng{/*fDeterministic=*/false};
+    for (size_t test{0}; test < 100; ++test) {
+        const size_t write_size{1 + rng.randrange(100U)};
+        const size_t key_offset{rng.randrange(3 * 8U)}; // Should wrap around
+
+
+        const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
+        const Obfuscation obfuscation{key_bytes};
+        std::vector expected{rng.randbytes<std::byte>(write_size)};
+        std::vector actual{expected};
+
+        expected_xor(expected, key_bytes, key_offset);
+        obfuscation(actual, key_offset);
+
+        BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end());
+    }
+}
+
+BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
+{
+    auto apply_random_xor_chunks{[](std::span<std::byte> write, const Obfuscation& obfuscation, FastRandomContext& rng) {
+        for (size_t offset{0}; offset < write.size();) {
+            const size_t chunk_size{1 + rng.randrange(write.size() - offset)};
+            obfuscation(write.subspan(offset, chunk_size), offset);
+            offset += chunk_size;
+        }
+    }};
+
+    FastRandomContext rng{/*fDeterministic=*/false};
+    for (size_t test{0}; test < 100; ++test) {
+        const size_t write_size{1 + rng.randrange(100U)};
+        const std::vector original{rng.randbytes<std::byte>(write_size)};
+        std::vector roundtrip{original};
+
+        const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
+        const Obfuscation obfuscation{key_bytes};
+        apply_random_xor_chunks(roundtrip, obfuscation, rng);
+
+        const bool all_zero = !obfuscation || (HexStr(key_bytes).find_first_not_of('0') >= write_size * 2);
+        BOOST_CHECK_EQUAL(original != roundtrip, !all_zero);
+
+        apply_random_xor_chunks(roundtrip, obfuscation, rng);
+        BOOST_CHECK(original == roundtrip);
+    }
+}
+
 BOOST_AUTO_TEST_CASE(xor_file)
 {
     fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
     auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
     const std::vector<uint8_t> test1{1, 2, 3};
     const std::vector<uint8_t> test2{4, 5};
-    const std::vector<std::byte> xor_pat{std::byte{0xff}, std::byte{0x00}};
+    constexpr std::array xor_pat{std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}};
+    uint64_t xor_key;
+    std::memcpy(&xor_key, xor_pat.data(), sizeof xor_key);
+
     {
         // Check errors for missing file
-        AutoFile xor_file{raw_file("rb"), xor_pat};
+        AutoFile xor_file{raw_file("rb"), xor_key};
         BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"});
         BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"});
         BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"});
@@ -35,7 +140,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
 #else
         const char* mode = "wbx";
 #endif
-        AutoFile xor_file{raw_file(mode), xor_pat};
+        AutoFile xor_file{raw_file(mode), xor_key};
         xor_file << test1 << test2;
     }
     {
@@ -48,7 +153,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
         BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
     }
     {
-        AutoFile xor_file{raw_file("rb"), xor_pat};
+        AutoFile xor_file{raw_file("rb"), xor_key};
         std::vector<std::byte> read1, read2;
         xor_file >> read1 >> read2;
         BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
@@ -57,7 +162,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
         BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
     }
     {
-        AutoFile xor_file{raw_file("rb"), xor_pat};
+        AutoFile xor_file{raw_file("rb"), xor_key};
         std::vector<std::byte> read2;
         // Check that ignore works
         xor_file.ignore(4);
@@ -73,7 +178,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer)
 {
     unsigned char a(1);
     unsigned char b(2);
-    unsigned char bytes[] = { 3, 4, 5, 6 };
+    unsigned char bytes[] = {3, 4, 5, 6};
     std::vector<unsigned char> vch;
 
     // Each test runs twice. Serializing a second time at the same starting
@@ -225,29 +330,30 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
     // Degenerate case
     {
         DataStream ds{in};
-        ds.Xor({0x00, 0x00});
+        Obfuscation{0}(ds);
         BOOST_CHECK_EQUAL(""s, ds.str());
     }
 
     in.push_back(std::byte{0x0f});
     in.push_back(std::byte{0xf0});
 
-    // Single character key
     {
+        const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}}};
+
         DataStream ds{in};
-        ds.Xor({0xff});
+        obfuscation(ds);
         BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
     }
 
-    // Multi character key
-
     in.clear();
     in.push_back(std::byte{0xf0});
     in.push_back(std::byte{0x0f});
 
     {
+        const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}}};
+
         DataStream ds{in};
-        ds.Xor({0xff, 0x0f});
+        obfuscation(ds);
         BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
     }
 }
@@ -270,7 +376,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
         BOOST_CHECK(false);
     } catch (const std::exception& e) {
         BOOST_CHECK(strstr(e.what(),
-                        "Rewind limit must be less than buffer size") != nullptr);
+                           "Rewind limit must be less than buffer size") != nullptr);
     }
 
     // The buffer is 25 bytes, allow rewinding 10 bytes.
@@ -359,7 +465,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
         BOOST_CHECK(false);
     } catch (const std::exception& e) {
         BOOST_CHECK(strstr(e.what(),
-                        "BufferedFile::Fill: end of file") != nullptr);
+                           "BufferedFile::Fill: end of file") != nullptr);
     }
     // Attempting to read beyond the end sets the EOF indicator.
     BOOST_CHECK(bf.eof());
@@ -567,4 +673,29 @@ BOOST_AUTO_TEST_CASE(streams_hashed)
     BOOST_CHECK_EQUAL(hash_writer.GetHash(), hash_verifier.GetHash());
 }
 
+BOOST_AUTO_TEST_CASE(streams_datastream_write_large)
+{
+    const uint32_t v1{m_rng.rand32()}, v2{m_rng.rand32()};
+    const fs::path tmp_path{m_args.GetDataDirBase() / "test_datastream_write_large.bin"};
+
+    // Write out the values through in a precisely sized vector.
+    {
+        AutoFile file{fsbridge::fopen(tmp_path, "w+b")};
+        DataStream data_stream(sizeof(v1) + sizeof(v2));
+        assert(data_stream.empty());
+        file.write_large(data_stream << v1 << v2);
+    }
+
+    // Read back and verify.
+    {
+        AutoFile file{fsbridge::fopen(tmp_path, "rb")};
+        uint32_t v3{0}, v4{0};
+        file >> v3 >> v4;
+        BOOST_CHECK_EQUAL(v3, v1);
+        BOOST_CHECK_EQUAL(v4, v2);
+    }
+
+    fs::remove(tmp_path);
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/test/transaction_tests.cpp

@@ -406,20 +406,110 @@ BOOST_AUTO_TEST_CASE(tx_oversized)
     }
 }
 
-BOOST_AUTO_TEST_CASE(basic_transaction_tests)
+static CMutableTransaction CreateTransaction()
 {
-    // Random real transaction (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436)
-    unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00};
-    std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1);
-    DataStream stream(vch);
+    // Serialized random real transaction (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436)
+    static constexpr auto ser_tx{"01000000016bff7fcd4f8565ef406dd5d63d4ff94f318fe82027fd4dc451b04474019f74b4000000008c493046022100da0dc6aecefe1e06efdf05773757deb168820930e3b0d03f46f5fcf150bf990c022100d25b5c87040076e4f253f8262e763e2dd51e7ff0be157727c4bc42807f17bd39014104e6c26ef67dc610d2cd192484789a6cf9aea9930b944b7e2db5342b9d9e5b9ff79aff9a2ee1978dd7fd01dfc522ee02283d3b06a9d03acf8096968d7dbb0f9178ffffffff028ba7940e000000001976a914badeecfdef0507247fc8f74241d73bc039972d7b88ac4094a802000000001976a914c10932483fec93ed51f5fe95e72559f2cc7043f988ac0000000000"_hex};
     CMutableTransaction tx;
-    stream >> TX_WITH_WITNESS(tx);
+    DataStream(ser_tx) >> TX_WITH_WITNESS(tx);
+    return tx;
+}
+
+BOOST_AUTO_TEST_CASE(transaction_duplicate_input_test)
+{
+    auto tx{CreateTransaction()};
+
     TxValidationState state;
     BOOST_CHECK_MESSAGE(CheckTransaction(CTransaction(tx), state) && state.IsValid(), "Simple deserialized transaction should be valid.");
 
-    // Check that duplicate txins fail
-    tx.vin.push_back(tx.vin[0]);
-    BOOST_CHECK_MESSAGE(!CheckTransaction(CTransaction(tx), state) || !state.IsValid(), "Transaction with duplicate txins should be invalid.");
+    // Add duplicate input
+    tx.vin.emplace_back(tx.vin[0]);
+    std::ranges::shuffle(tx.vin, m_rng);
+    BOOST_CHECK_MESSAGE(!CheckTransaction(CTransaction(tx), state) || !state.IsValid(), "Transaction with 2 duplicate txins should be invalid.");
+
+    // ... add a valid input for more complex check
+    tx.vin.emplace_back(COutPoint(Txid::FromUint256(uint256{1}), 1));
+    std::ranges::shuffle(tx.vin, m_rng);
+    BOOST_CHECK_MESSAGE(!CheckTransaction(CTransaction(tx), state) || !state.IsValid(), "Transaction with 3 inputs (2 valid, 1 duplicate) should be invalid.");
+}
+
+BOOST_AUTO_TEST_CASE(transaction_duplicate_detection_test)
+{
+    // Randomized testing against hash- and tree-based duplicate check
+    auto reference_duplicate_check_hash{[](const std::vector<CTxIn>& vin) {
+        std::unordered_set<COutPoint, SaltedOutpointHasher> vInOutPoints;
+        for (const auto& txin : vin) {
+            if (!vInOutPoints.insert(txin.prevout).second) {
+                return false;
+            }
+        }
+        return true;
+    }};
+    auto reference_duplicate_check_tree{[](const std::vector<CTxIn>& vin) {
+        std::set<COutPoint> vInOutPoints;
+        for (const auto& txin : vin) {
+            if (!vInOutPoints.insert(txin.prevout).second) {
+                return false;
+            }
+        }
+        return true;
+    }};
+
+    std::vector<Txid> hashes;
+    std::vector<uint32_t> ns;
+    for (int i = 0; i < 10; ++i) {
+        hashes.emplace_back(Txid::FromUint256(m_rng.rand256()));
+        ns.emplace_back(m_rng.rand32());
+    }
+    auto tx{CreateTransaction()};
+    TxValidationState state;
+    for (int i{0}; i < 100; ++i) {
+        if (m_rng.randbool()) {
+            tx.vin.clear();
+        }
+        for (int j{0}, num_inputs{1 + m_rng.randrange(5)}; j < num_inputs; ++j) {
+            if (COutPoint outpoint(hashes[m_rng.randrange(hashes.size())], ns[m_rng.randrange(ns.size())]); !outpoint.IsNull()) {
+                tx.vin.emplace_back(outpoint);
+            }
+        }
+        std::ranges::shuffle(tx.vin, m_rng);
+
+        bool actual{CheckTransaction(CTransaction(tx), state)};
+        BOOST_CHECK_EQUAL(actual, reference_duplicate_check_hash(tx.vin));
+        BOOST_CHECK_EQUAL(actual, reference_duplicate_check_tree(tx.vin));
+    }
+}
+
+BOOST_AUTO_TEST_CASE(transaction_null_prevout_detection_test)
+{
+    // Randomized testing against linear null prevout check
+    auto reference_null_prevout_check_hash{[](const std::vector<CTxIn>& vin) {
+        for (const auto& txin : vin) {
+            if (txin.prevout.IsNull()) {
+                return false;
+            }
+        }
+        return true;
+    }};
+
+    auto tx{CreateTransaction()};
+    TxValidationState state;
+    for (int i{0}; i < 100; ++i) {
+        if (m_rng.randbool()) {
+            tx.vin.clear();
+        }
+        for (int j{0}, num_inputs{1 + m_rng.randrange(5)}; j < num_inputs; ++j) {
+            switch (m_rng.randrange(5)) {
+            case 0: tx.vin.emplace_back(COutPoint()); break;                                               // Null prevout
+            case 1: tx.vin.emplace_back(Txid::FromUint256(uint256::ZERO), m_rng.rand32()); break;          // Null hash, random index
+            case 2: tx.vin.emplace_back(Txid::FromUint256(m_rng.rand256()), COutPoint::NULL_INDEX); break; // Random hash, Null index
+            default: tx.vin.emplace_back(Txid::FromUint256(m_rng.rand256()), m_rng.rand32());              // Random prevout
+            }
+        }
+        std::ranges::shuffle(tx.vin, m_rng);
+
+        BOOST_CHECK_EQUAL(CheckTransaction(CTransaction(tx), state), reference_null_prevout_check_hash(tx.vin));
+    }
 }
 
 BOOST_AUTO_TEST_CASE(test_Get)
@@ -1048,4 +1138,116 @@ BOOST_AUTO_TEST_CASE(test_IsStandard)
     CheckIsNotStandard(t, "dust");
 }
 
+BOOST_AUTO_TEST_CASE(test_uint256_sorting)
+{
+    // Sorting
+    std::vector original{
+        uint256{1},
+        uint256{2},
+        uint256{3}
+    };
+
+    std::vector shuffled{original};
+    std::ranges::shuffle(shuffled, m_rng);
+    std::sort(shuffled.begin(), shuffled.end());
+
+    BOOST_CHECK_EQUAL_COLLECTIONS(original.begin(), original.end(), shuffled.begin(), shuffled.end());
+
+    // Operators
+    constexpr auto a{uint256{1}},
+                   b{uint256{2}},
+                   c{uint256{3}};
+
+    BOOST_CHECK(a == a);
+    BOOST_CHECK(a == uint256{1});
+    BOOST_CHECK(b == b);
+    BOOST_CHECK(c == c);
+    BOOST_CHECK(a != b);
+    BOOST_CHECK(a != uint256{10});
+    BOOST_CHECK(a != c);
+    BOOST_CHECK(b != c);
+
+    BOOST_CHECK(a < b);
+    BOOST_CHECK(a < uint256{10});
+    BOOST_CHECK(b < c);
+    BOOST_CHECK(a < c);
+}
+
+BOOST_AUTO_TEST_CASE(test_transaction_identifier_sorting)
+{
+    std::vector original{
+        Txid::FromUint256(uint256{1}),
+        Txid::FromUint256(uint256{2}),
+        Txid::FromUint256(uint256{3})
+    };
+
+    std::vector shuffled{original};
+    std::ranges::shuffle(shuffled, m_rng);
+    std::sort(shuffled.begin(), shuffled.end());
+
+    BOOST_CHECK_EQUAL_COLLECTIONS(original.begin(), original.end(), shuffled.begin(), shuffled.end());
+
+    // Operators
+    const auto a(Txid::FromUint256(uint256{1})),
+               b(Txid::FromUint256(uint256{2})),
+               c(Txid::FromUint256(uint256{3}));
+
+    BOOST_CHECK(a == uint256{1});
+
+    BOOST_CHECK(a == a);
+    BOOST_CHECK(a == Txid::FromUint256(uint256{1}));
+    BOOST_CHECK(b == b);
+    BOOST_CHECK(c == c);
+    BOOST_CHECK(a != b);
+    BOOST_CHECK(a != Txid::FromUint256(uint256{10}));
+    BOOST_CHECK(a != c);
+    BOOST_CHECK(b != c);
+
+    BOOST_CHECK(a < b);
+    BOOST_CHECK(a < Txid::FromUint256(uint256{10}));
+    BOOST_CHECK(b < c);
+    BOOST_CHECK(a < c);
+}
+
+BOOST_AUTO_TEST_CASE(test_coutpoint_sorting)
+{
+    // Sorting
+    std::vector original{
+        COutPoint(Txid::FromUint256(uint256{1}), 1),
+        COutPoint(Txid::FromUint256(uint256{1}), 2),
+        COutPoint(Txid::FromUint256(uint256{1}), 3),
+        COutPoint(Txid::FromUint256(uint256{2}), 1),
+        COutPoint(Txid::FromUint256(uint256{2}), 2),
+        COutPoint(Txid::FromUint256(uint256{2}), 3),
+        COutPoint(Txid::FromUint256(uint256{3}), 1),
+        COutPoint(Txid::FromUint256(uint256{3}), 2),
+        COutPoint(Txid::FromUint256(uint256{3}), 3)
+    };
+
+    std::vector shuffled{original};
+    std::ranges::shuffle(shuffled, m_rng);
+    std::sort(shuffled.begin(), shuffled.end());
+
+    BOOST_CHECK_EQUAL_COLLECTIONS(original.begin(), original.end(), shuffled.begin(), shuffled.end());
+
+    // Operators
+    const auto a{COutPoint(Txid::FromUint256(uint256{1}), 1)},
+               b{COutPoint(Txid::FromUint256(uint256{1}), 2)},
+               c{COutPoint(Txid::FromUint256(uint256{2}), 1)};
+
+    BOOST_CHECK(a == a);
+    BOOST_CHECK(a == COutPoint(Txid::FromUint256(uint256{1}), 1));
+    BOOST_CHECK(b == b);
+    BOOST_CHECK(c == c);
+    BOOST_CHECK(a != b);
+    BOOST_CHECK(a != COutPoint(Txid::FromUint256(uint256{1}), 10));
+    BOOST_CHECK(a != c);
+    BOOST_CHECK(b != c);
+
+    BOOST_CHECK(a < b);
+    BOOST_CHECK(a < COutPoint(Txid::FromUint256(uint256{1}), 10));
+    BOOST_CHECK(b < c);
+    BOOST_CHECK(a < c);
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/test/util/setup_common.cpp

@@ -616,3 +616,8 @@ std::ostream& operator<<(std::ostream& os, const uint256& num)
 {
     return os << num.ToString();
 }
+
+std::ostream& operator<<(std::ostream& os, const COutPoint& outpoint)
+{
+    return os << outpoint.hash << ", " << outpoint.n;
+}

src/test/util/setup_common.h

@@ -291,6 +291,7 @@ inline std::ostream& operator<<(std::ostream& os, const std::optional<T>& v)
 std::ostream& operator<<(std::ostream& os, const arith_uint256& num);
 std::ostream& operator<<(std::ostream& os, const uint160& num);
 std::ostream& operator<<(std::ostream& os, const uint256& num);
+std::ostream& operator<<(std::ostream& os, const COutPoint& outpoint);
 // @}
 
 /**

src/txdb.h

@@ -22,7 +22,7 @@ class COutPoint;
 class uint256;
 
 //! -dbbatchsize default (bytes)
-static const int64_t nDefaultDbBatchSize = 16 << 20;
+static const int64_t nDefaultDbBatchSize = 64 << 20;
 
 //! User-controlled performance and debug options.
 struct CoinsViewOptions {

src/util/hasher.cpp

@@ -11,9 +11,9 @@ SaltedTxidHasher::SaltedTxidHasher() :
     k0{FastRandomContext().rand64()},
     k1{FastRandomContext().rand64()} {}
 
-SaltedOutpointHasher::SaltedOutpointHasher(bool deterministic) :
-    k0{deterministic ? 0x8e819f2607a18de6 : FastRandomContext().rand64()},
-    k1{deterministic ? 0xf4020d2e3983b0eb : FastRandomContext().rand64()}
+SaltedOutpointHasher::SaltedOutpointHasher(bool deterministic) : hasher{
+    deterministic ? 0x8e819f2607a18de6 : FastRandomContext().rand64(),
+    deterministic ? 0xf4020d2e3983b0eb : FastRandomContext().rand64()}
 {}
 
 SaltedSipHasher::SaltedSipHasher() :

src/util/hasher.h

@@ -30,12 +30,10 @@ public:
 
 class SaltedOutpointHasher
 {
-private:
-    /** Salt */
-    const uint64_t k0, k1;
+    const Uint256ExtraSipHasher hasher;
 
 public:
-    SaltedOutpointHasher(bool deterministic = false);
+    explicit SaltedOutpointHasher(bool deterministic = false);
 
     /**
      * Having the hash noexcept allows libstdc++'s unordered_map to recalculate
@@ -47,7 +45,7 @@ public:
      * @see https://gcc.gnu.org/onlinedocs/gcc-13.2.0/libstdc++/manual/manual/unordered_associative.html
      */
     size_t operator()(const COutPoint& id) const noexcept {
-        return SipHashUint256Extra(k0, k1, id.hash, id.n);
+        return hasher(id.hash, id.n);
     }
 };
 

test/functional/feature_init.py

@@ -88,7 +88,7 @@ class InitTest(BitcoinTestFramework):
 
         args = ['-txindex=1', '-blockfilterindex=1', '-coinstatsindex=1']
         for terminate_line in lines_to_terminate_after:
-            self.log.info(f"Starting node and will exit after line {terminate_line}")
+            self.log.info(f"Starting node and will terminate after line {terminate_line}")
             with node.busy_wait_for_debug_log([terminate_line]):
                 if platform.system() == 'Windows':
                     # CREATE_NEW_PROCESS_GROUP is required in order to be able
@@ -108,12 +108,22 @@ class InitTest(BitcoinTestFramework):
             'blocks/index/*.ldb': 'Error opening block database.',
             'chainstate/*.ldb': 'Error opening coins database.',
             'blocks/blk*.dat': 'Error loading block database.',
+            'indexes/txindex/MANIFEST*': 'LevelDB error: Corruption: CURRENT points to a non-existent file',
+            # Removing these files does not result in a startup error:
+            # 'indexes/blockfilter/basic/*.dat', 'indexes/blockfilter/basic/db/*.*', 'indexes/coinstats/db/*.*',
+            # 'indexes/txindex/*.log', 'indexes/txindex/CURRENT', 'indexes/txindex/LOCK'
         }
 
         files_to_perturb = {
             'blocks/index/*.ldb': 'Error loading block database.',
             'chainstate/*.ldb': 'Error opening coins database.',
             'blocks/blk*.dat': 'Corrupted block database detected.',
+            'indexes/blockfilter/basic/db/*.*': 'LevelDB error: Corruption',
+            'indexes/coinstats/db/*.*': 'LevelDB error: Corruption',
+            'indexes/txindex/*.log': 'LevelDB error: Corruption',
+            'indexes/txindex/CURRENT': 'LevelDB error: Corruption',
+            # Perturbing these files does not result in a startup error:
+            # 'indexes/blockfilter/basic/*.dat', 'indexes/txindex/MANIFEST*', 'indexes/txindex/LOCK'
         }
 
         for file_patt, err_fragment in files_to_delete.items():
@@ -135,9 +145,10 @@ class InitTest(BitcoinTestFramework):
             self.stop_node(0)
 
         self.log.info("Test startup errors after perturbing certain essential files")
+        dirs = ["blocks", "chainstate", "indexes"]
         for file_patt, err_fragment in files_to_perturb.items():
-            shutil.copytree(node.chain_path / "blocks", node.chain_path / "blocks_bak")
-            shutil.copytree(node.chain_path / "chainstate", node.chain_path / "chainstate_bak")
+            for dir in dirs:
+                shutil.copytree(node.chain_path / dir, node.chain_path / f"{dir}_bak")
             target_files = list(node.chain_path.glob(file_patt))
 
             for target_file in target_files:
@@ -151,10 +162,9 @@ class InitTest(BitcoinTestFramework):
 
             start_expecting_error(err_fragment)
 
-            shutil.rmtree(node.chain_path / "blocks")
-            shutil.rmtree(node.chain_path / "chainstate")
-            shutil.move(node.chain_path / "blocks_bak", node.chain_path / "blocks")
-            shutil.move(node.chain_path / "chainstate_bak", node.chain_path / "chainstate")
+            for dir in dirs:
+                shutil.rmtree(node.chain_path / dir)
+                shutil.move(node.chain_path / f"{dir}_bak", node.chain_path / dir)
 
     def init_pid_test(self):
         BITCOIN_PID_FILENAME_CUSTOM = "my_fancy_bitcoin_pid_file.foobar"

test/functional/mempool_accept.py

@@ -45,6 +45,7 @@ from test_framework.util import (
     assert_equal,
     assert_greater_than,
     assert_raises_rpc_error,
+    sync_txindex,
 )
 from test_framework.wallet import MiniWallet
 from test_framework.wallet_util import generate_keypair
@@ -270,6 +271,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
 
         self.log.info('A coinbase transaction')
         # Pick the input of the first tx we created, so it has to be a coinbase tx
+        sync_txindex(self, node)
         raw_tx_coinbase_spent = node.getrawtransaction(txid=node.decoderawtransaction(hexstring=raw_tx_in_block)['vin'][0]['txid'])
         tx = tx_from_hex(raw_tx_coinbase_spent)
         self.check_mempool_result(

test/functional/rpc_rawtransaction.py

@@ -34,6 +34,7 @@ from test_framework.util import (
     assert_equal,
     assert_greater_than,
     assert_raises_rpc_error,
+    sync_txindex,
 )
 from test_framework.wallet import (
     getnewdestination,
@@ -70,7 +71,7 @@ class RawTransactionsTest(BitcoinTestFramework):
         self.num_nodes = 3
         self.extra_args = [
             ["-txindex"],
-            ["-txindex"],
+            [],
             ["-fastprune", "-prune=1"],
         ]
         # whitelist peers to speed up tx relay / mempool sync
@@ -109,6 +110,7 @@ class RawTransactionsTest(BitcoinTestFramework):
             self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex")
 
             if n == 0:
+                sync_txindex(self, self.nodes[n])
                 # With -txindex.
                 # 1. valid parameters - only supply txid
                 assert_equal(self.nodes[n].getrawtransaction(txId), tx['hex'])

test/functional/rpc_txoutproof.py

@@ -12,6 +12,7 @@ from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
     assert_raises_rpc_error,
+    sync_txindex,
 )
 from test_framework.wallet import MiniWallet
 
@@ -77,6 +78,7 @@ class MerkleBlockTest(BitcoinTestFramework):
         assert_equal(sorted(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2]))), sorted(txlist))
         assert_equal(sorted(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid2, txid1]))), sorted(txlist))
         # We can always get a proof if we have a -txindex
+        sync_txindex(self, self.nodes[1])
         assert_equal(self.nodes[0].verifytxoutproof(self.nodes[1].gettxoutproof([txid_spent])), [txid_spent])
         # We can't get a proof if we specify transactions from different blocks
         assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[0].gettxoutproof, [txid1, txid3])

test/functional/test_framework/util.py

@@ -592,3 +592,10 @@ def find_vout_for_address(node, txid, addr):
         if addr == tx["vout"][i]["scriptPubKey"]["address"]:
             return i
     raise RuntimeError("Vout not found for address: txid=%s, addr=%s" % (txid, addr))
+
+
+def sync_txindex(test_framework, node):
+    test_framework.log.debug("Waiting for node txindex to sync")
+    sync_start = int(time.time())
+    test_framework.wait_until(lambda: node.getindexinfo("txindex")["txindex"]["synced"])
+    test_framework.log.debug(f"Synced in {time.time() - sync_start} seconds")

test/functional/wallet_avoid_mixing_output_types.py

@@ -117,7 +117,6 @@ class AddressInputTypeGrouping(BitcoinTestFramework):
         self.extra_args = [
             [
                 "-addresstype=bech32",
-                "-txindex",
             ],
             [
                 "-addresstype=p2sh-segwit",