mirror of
https://github.com/bitcoin/bitcoin.git
synced 2025-04-08 03:59:18 +02:00
Merge d950fc6f4999191476bdf6cab1793f6534cf1075 into db2c57ae9eebdb75c58cd165ac929919969c19a9
This commit is contained in:
l0rinc
GitHub
commit
5a15b0dd35
@ -7,17 +7,23 @@
|
||||
#include <span.h>
|
||||
#include <streams.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
static void Xor(benchmark::Bench& bench)
|
||||
{
|
||||
FastRandomContext frc{/*fDeterministic=*/true};
|
||||
auto data{frc.randbytes<std::byte>(1024)};
|
||||
auto key{frc.randbytes<std::byte>(31)};
|
||||
FastRandomContext rng{/*fDeterministic=*/true};
|
||||
auto test_data{rng.randbytes<std::byte>(1 << 20)};
|
||||
|
||||
bench.batch(data.size()).unit("byte").run([&] {
|
||||
util::Xor(data, key);
|
||||
const Obfuscation obfuscation{rng.rand64()};
|
||||
assert(obfuscation);
|
||||
|
||||
size_t offset{0};
|
||||
bench.batch(test_data.size()).unit("byte").run([&] {
|
||||
obfuscation(test_data, offset++);
|
||||
ankerl::nanobench::doNotOptimizeAway(test_data);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -171,7 +171,7 @@ void CDBBatch::Clear()
|
||||
void CDBBatch::WriteImpl(Span<const std::byte> key, DataStream& ssValue)
|
||||
{
|
||||
leveldb::Slice slKey(CharCast(key.data()), key.size());
|
||||
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
|
||||
dbwrapper_private::GetObfuscation(parent)(ssValue);
|
||||
leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
|
||||
m_impl_batch->batch.Put(slKey, slValue);
|
||||
// LevelDB serializes writes as:
|
||||
@ -220,7 +220,11 @@ struct LevelDBContext {
|
||||
};
|
||||
|
||||
CDBWrapper::CDBWrapper(const DBParams& params)
|
||||
: m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
|
||||
: m_db_context{std::make_unique<LevelDBContext>()},
|
||||
m_name{fs::PathToString(params.path.stem())},
|
||||
m_obfuscation{0},
|
||||
m_path{params.path},
|
||||
m_is_memory{params.memory_only}
|
||||
{
|
||||
DBContext().penv = nullptr;
|
||||
DBContext().readoptions.verify_checksums = true;
|
||||
@ -255,24 +259,23 @@ CDBWrapper::CDBWrapper(const DBParams& params)
|
||||
LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
|
||||
}
|
||||
|
||||
// The base-case obfuscation key, which is a noop.
|
||||
obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
|
||||
|
||||
bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
|
||||
|
||||
if (!key_exists && params.obfuscate && IsEmpty()) {
|
||||
// Initialize non-degenerate obfuscation if it won't upset
|
||||
// existing, non-obfuscated data.
|
||||
std::vector<unsigned char> new_key = CreateObfuscateKey();
|
||||
m_obfuscation = 0; // Needed for unobfuscated Read
|
||||
std::vector<unsigned char> obfuscate_key_vector(Obfuscation::SIZE_BYTES, '\000');
|
||||
const bool key_missing = !Read(OBFUSCATE_KEY_KEY, obfuscate_key_vector);
|
||||
if (key_missing && params.obfuscate && IsEmpty()) {
|
||||
// Initialize non-degenerate obfuscation if it won't upset existing, non-obfuscated data.
|
||||
std::vector<uint8_t> new_key(Obfuscation::SIZE_BYTES);
|
||||
GetRandBytes(new_key);
|
||||
|
||||
// Write `new_key` so we don't obfuscate the key with itself
|
||||
Write(OBFUSCATE_KEY_KEY, new_key);
|
||||
obfuscate_key = new_key;
|
||||
obfuscate_key_vector = new_key;
|
||||
|
||||
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
|
||||
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
|
||||
}
|
||||
|
||||
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
|
||||
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
|
||||
m_obfuscation = obfuscate_key_vector;
|
||||
obfuscate_key_vector.clear();
|
||||
}
|
||||
|
||||
CDBWrapper::~CDBWrapper()
|
||||
@ -323,19 +326,6 @@ size_t CDBWrapper::DynamicMemoryUsage() const
|
||||
// past the null-terminator.
|
||||
const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
|
||||
|
||||
const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
|
||||
|
||||
/**
|
||||
* Returns a string (consisting of 8 random bytes) suitable for use as an
|
||||
* obfuscating XOR key.
|
||||
*/
|
||||
std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
|
||||
{
|
||||
std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
|
||||
GetRandBytes(ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
|
||||
{
|
||||
leveldb::Slice slKey(CharCast(key.data()), key.size());
|
||||
@ -418,10 +408,5 @@ void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
|
||||
void CDBIterator::Next() { m_impl_iter->iter->Next(); }
|
||||
|
||||
namespace dbwrapper_private {
|
||||
|
||||
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
|
||||
{
|
||||
return w.obfuscate_key;
|
||||
}
|
||||
|
||||
Obfuscation GetObfuscation(const CDBWrapper& w) { return w.m_obfuscation; }
|
||||
} // namespace dbwrapper_private
|
||||
|
||||
@ -63,8 +63,7 @@ namespace dbwrapper_private {
|
||||
* Database obfuscation should be considered an implementation detail of the
|
||||
* specific database.
|
||||
*/
|
||||
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
|
||||
|
||||
Obfuscation GetObfuscation(const CDBWrapper&);
|
||||
}; // namespace dbwrapper_private
|
||||
|
||||
bool DestroyDB(const std::string& path_str);
|
||||
@ -168,7 +167,7 @@ public:
|
||||
template<typename V> bool GetValue(V& value) {
|
||||
try {
|
||||
DataStream ssValue{GetValueImpl()};
|
||||
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
|
||||
dbwrapper_private::GetObfuscation(parent)(ssValue);
|
||||
ssValue >> value;
|
||||
} catch (const std::exception&) {
|
||||
return false;
|
||||
@ -181,7 +180,7 @@ struct LevelDBContext;
|
||||
|
||||
class CDBWrapper
|
||||
{
|
||||
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
|
||||
friend Obfuscation dbwrapper_private::GetObfuscation(const CDBWrapper&);
|
||||
private:
|
||||
//! holds all leveldb-specific fields of this class
|
||||
std::unique_ptr<LevelDBContext> m_db_context;
|
||||
@ -190,16 +189,11 @@ private:
|
||||
std::string m_name;
|
||||
|
||||
//! a key used for optional XOR-obfuscation of the database
|
||||
std::vector<unsigned char> obfuscate_key;
|
||||
Obfuscation m_obfuscation;
|
||||
|
||||
//! the key under which the obfuscation key is stored
|
||||
static const std::string OBFUSCATE_KEY_KEY;
|
||||
|
||||
//! the length of the obfuscate key in number of bytes
|
||||
static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
|
||||
|
||||
std::vector<unsigned char> CreateObfuscateKey() const;
|
||||
|
||||
//! path to filesystem storage
|
||||
const fs::path m_path;
|
||||
|
||||
@ -230,7 +224,7 @@ public:
|
||||
}
|
||||
try {
|
||||
DataStream ssValue{MakeByteSpan(*strValue)};
|
||||
ssValue.Xor(obfuscate_key);
|
||||
m_obfuscation(ssValue);
|
||||
ssValue >> value;
|
||||
} catch (const std::exception&) {
|
||||
return false;
|
||||
|
||||
@ -774,13 +774,13 @@ void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
|
||||
|
||||
AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
|
||||
{
|
||||
return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key};
|
||||
return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation};
|
||||
}
|
||||
|
||||
/** Open an undo file (rev?????.dat) */
|
||||
AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
|
||||
{
|
||||
return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key};
|
||||
return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation};
|
||||
}
|
||||
|
||||
fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
|
||||
@ -1097,7 +1097,7 @@ FlatFilePos BlockManager::WriteBlock(const CBlock& block, int nHeight)
|
||||
return pos;
|
||||
}
|
||||
|
||||
static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
|
||||
static Obfuscation InitBlocksdirXorKey(const BlockManager::Options& opts)
|
||||
{
|
||||
// Bytes are serialized without length indicator, so this is also the exact
|
||||
// size of the XOR-key file.
|
||||
@ -1146,12 +1146,12 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
|
||||
};
|
||||
}
|
||||
LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
|
||||
return std::vector<std::byte>{xor_key.begin(), xor_key.end()};
|
||||
return Obfuscation{xor_key};
|
||||
}
|
||||
|
||||
BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
|
||||
: m_prune_mode{opts.prune_target > 0},
|
||||
m_xor_key{InitBlocksdirXorKey(opts)},
|
||||
m_obfuscation{InitBlocksdirXorKey(opts)},
|
||||
m_opts{std::move(opts)},
|
||||
m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
|
||||
m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
|
||||
|
||||
@ -235,7 +235,7 @@ private:
|
||||
|
||||
const bool m_prune_mode;
|
||||
|
||||
const std::vector<std::byte> m_xor_key;
|
||||
const Obfuscation m_obfuscation;
|
||||
|
||||
/** Dirty block index entries. */
|
||||
std::set<CBlockIndex*> m_dirty_blockindex;
|
||||
|
||||
@ -58,15 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
|
||||
try {
|
||||
uint64_t version;
|
||||
file >> version;
|
||||
std::vector<std::byte> xor_key;
|
||||
if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
|
||||
// Leave XOR-key empty
|
||||
file.SetObfuscation(0);
|
||||
} else if (version == MEMPOOL_DUMP_VERSION) {
|
||||
file >> xor_key;
|
||||
Obfuscation obfuscation{0};
|
||||
file >> obfuscation;
|
||||
file.SetObfuscation(obfuscation);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
file.SetXor(xor_key);
|
||||
uint64_t total_txns_to_load;
|
||||
file >> total_txns_to_load;
|
||||
uint64_t txns_tried = 0;
|
||||
@ -177,12 +177,13 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock
|
||||
const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION};
|
||||
file << version;
|
||||
|
||||
std::vector<std::byte> xor_key(8);
|
||||
if (!pool.m_opts.persist_v1_dat) {
|
||||
FastRandomContext{}.fillrand(xor_key);
|
||||
file << xor_key;
|
||||
const Obfuscation obfuscation{FastRandomContext{}.rand64()};
|
||||
file << obfuscation;
|
||||
file.SetObfuscation(obfuscation);
|
||||
} else {
|
||||
file.SetObfuscation(0);
|
||||
}
|
||||
file.SetXor(xor_key);
|
||||
|
||||
uint64_t mempool_transactions_to_write(vinfo.size());
|
||||
file << mempool_transactions_to_write;
|
||||
|
||||
86
src/obfuscation.h
Normal file
86
src/obfuscation.h
Normal file
@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2009-present The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#ifndef BITCOIN_OBFUSCATION_H
|
||||
#define BITCOIN_OBFUSCATION_H
|
||||
|
||||
#include <array>
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <random>
|
||||
#include <span.h>
|
||||
#include <util/check.h>
|
||||
#include <cstring>
|
||||
#include <climits>
|
||||
#include <serialize.h>
|
||||
|
||||
class Obfuscation
|
||||
{
|
||||
public:
|
||||
static constexpr size_t SIZE_BYTES{sizeof(uint64_t)};
|
||||
|
||||
private:
|
||||
std::array<uint64_t, SIZE_BYTES> rotations; // Cached key rotations
|
||||
void SetRotations(const uint64_t key)
|
||||
{
|
||||
for (size_t i{0}; i < SIZE_BYTES; ++i)
|
||||
{
|
||||
size_t key_rotation_bits{CHAR_BIT * i};
|
||||
if constexpr (std::endian::native == std::endian::big) key_rotation_bits *= -1;
|
||||
rotations[i] = std::rotr(key, key_rotation_bits);
|
||||
}
|
||||
}
|
||||
|
||||
static uint64_t ToUint64(const Span<const std::byte> key_span)
|
||||
{
|
||||
uint64_t key{};
|
||||
std::memcpy(&key, key_span.data(), SIZE_BYTES);
|
||||
return key;
|
||||
}
|
||||
|
||||
static void Xor(Span<std::byte> write, const uint64_t key, const size_t size)
|
||||
{
|
||||
assert(size <= write.size());
|
||||
uint64_t raw{};
|
||||
std::memcpy(&raw, write.data(), size);
|
||||
raw ^= key;
|
||||
std::memcpy(write.data(), &raw, size);
|
||||
}
|
||||
|
||||
public:
|
||||
Obfuscation(const uint64_t key) { SetRotations(key); }
|
||||
Obfuscation(const Span<const std::byte> key_span) : Obfuscation(ToUint64(key_span)) {}
|
||||
Obfuscation(const std::array<const std::byte, SIZE_BYTES>& key_arr) : Obfuscation(ToUint64(key_arr)) {}
|
||||
Obfuscation(const std::vector<uint8_t>& key_vec) : Obfuscation(MakeByteSpan(key_vec)) {}
|
||||
|
||||
uint64_t Key() const { return rotations[0]; }
|
||||
operator bool() const { return Key() != 0; }
|
||||
void operator()(Span<std::byte> write, const size_t key_offset_bytes = 0) const
|
||||
{
|
||||
if (!*this) return;
|
||||
const uint64_t rot_key{rotations[key_offset_bytes % SIZE_BYTES]}; // Continue obfuscation from where we left off
|
||||
for (; write.size() >= SIZE_BYTES; write = write.subspan(SIZE_BYTES)) { // Process multiple bytes at a time
|
||||
Xor(write, rot_key, SIZE_BYTES);
|
||||
}
|
||||
Xor(write, rot_key, write.size());
|
||||
}
|
||||
|
||||
template <typename Stream>
|
||||
void Serialize(Stream& s) const
|
||||
{
|
||||
std::vector<std::byte> bytes(SIZE_BYTES);
|
||||
std::memcpy(bytes.data(), &rotations[0], SIZE_BYTES);
|
||||
s << bytes;
|
||||
}
|
||||
|
||||
template <typename Stream>
|
||||
void Unserialize(Stream& s)
|
||||
{
|
||||
std::vector<std::byte> bytes(SIZE_BYTES);
|
||||
s >> bytes;
|
||||
SetRotations(ToUint64(bytes));
|
||||
}
|
||||
};
|
||||
|
||||
#endif // BITCOIN_OBFUSCATION_H
|
||||
@ -9,8 +9,7 @@
|
||||
|
||||
#include <array>
|
||||
|
||||
AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
|
||||
: m_file{file}, m_xor{std::move(data_xor)}
|
||||
AutoFile::AutoFile(std::FILE* file, const Obfuscation& obfuscation) : m_file{file}, m_obfuscation{obfuscation}
|
||||
{
|
||||
if (!IsNull()) {
|
||||
auto pos{std::ftell(m_file)};
|
||||
@ -21,12 +20,12 @@ AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
|
||||
std::size_t AutoFile::detail_fread(Span<std::byte> dst)
|
||||
{
|
||||
if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr");
|
||||
size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
|
||||
if (!m_xor.empty()) {
|
||||
if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown");
|
||||
util::Xor(dst.subspan(0, ret), m_xor, *m_position);
|
||||
const size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
|
||||
if (m_obfuscation) {
|
||||
if (!m_position) throw std::ios_base::failure("AutoFile::read: position unknown");
|
||||
m_obfuscation(dst, *m_position);
|
||||
}
|
||||
if (m_position.has_value()) *m_position += ret;
|
||||
if (m_position) *m_position += ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -81,7 +80,7 @@ void AutoFile::ignore(size_t nSize)
|
||||
void AutoFile::write(Span<const std::byte> src)
|
||||
{
|
||||
if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr");
|
||||
if (m_xor.empty()) {
|
||||
if (!m_obfuscation) {
|
||||
if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
|
||||
throw std::ios_base::failure("AutoFile::write: write failed");
|
||||
}
|
||||
@ -91,8 +90,8 @@ void AutoFile::write(Span<const std::byte> src)
|
||||
std::array<std::byte, 4096> buf;
|
||||
while (src.size() > 0) {
|
||||
auto buf_now{Span{buf}.first(std::min<size_t>(src.size(), buf.size()))};
|
||||
std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin());
|
||||
util::Xor(buf_now, m_xor, *m_position);
|
||||
std::copy_n(src.begin(), buf_now.size(), buf_now.begin());
|
||||
m_obfuscation(buf_now, *m_position);
|
||||
if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) {
|
||||
throw std::ios_base::failure{"XorFile::write: failed"};
|
||||
}
|
||||
@ -102,6 +101,16 @@ void AutoFile::write(Span<const std::byte> src)
|
||||
}
|
||||
}
|
||||
|
||||
void AutoFile::write_large(Span<std::byte> src)
|
||||
{
|
||||
if (!m_file) throw std::ios_base::failure("AutoFile::write_large: file handle is nullptr");
|
||||
m_obfuscation(src, *m_position); // obfuscate in-place
|
||||
if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
|
||||
throw std::ios_base::failure("AutoFile::write_large: write failed");
|
||||
}
|
||||
if (m_position) *m_position += src.size();
|
||||
}
|
||||
|
||||
bool AutoFile::Commit()
|
||||
{
|
||||
return ::FileCommit(m_file);
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#ifndef BITCOIN_STREAMS_H
|
||||
#define BITCOIN_STREAMS_H
|
||||
|
||||
#include <obfuscation.h>
|
||||
#include <serialize.h>
|
||||
#include <span.h>
|
||||
#include <support/allocators/zeroafterfree.h>
|
||||
@ -21,30 +22,8 @@
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace util {
|
||||
inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_offset = 0)
|
||||
{
|
||||
if (key.size() == 0) {
|
||||
return;
|
||||
}
|
||||
key_offset %= key.size();
|
||||
|
||||
for (size_t i = 0, j = key_offset; i != write.size(); i++) {
|
||||
write[i] ^= key[j++];
|
||||
|
||||
// This potentially acts on very many bytes of data, so it's
|
||||
// important that we calculate `j`, i.e. the `key` index in this
|
||||
// way instead of doing a %, which would effectively be a division
|
||||
// for each byte Xor'd -- much slower than need be.
|
||||
if (j == key.size())
|
||||
j = 0;
|
||||
}
|
||||
}
|
||||
} // namespace util
|
||||
|
||||
/* Minimal stream for overwriting and/or appending to an existing byte vector
|
||||
*
|
||||
* The referenced vector will grow as necessary
|
||||
@ -261,21 +240,16 @@ public:
|
||||
return (*this);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
template <typename T>
|
||||
DataStream& operator>>(T&& obj)
|
||||
{
|
||||
::Unserialize(*this, obj);
|
||||
return (*this);
|
||||
}
|
||||
|
||||
/**
|
||||
* XOR the contents of this stream with a certain key.
|
||||
*
|
||||
* @param[in] key The key used to XOR the data in this stream.
|
||||
*/
|
||||
void Xor(const std::vector<unsigned char>& key)
|
||||
void Obfuscate(const Obfuscation& obfuscation)
|
||||
{
|
||||
util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key));
|
||||
if (obfuscation) obfuscation(MakeWritableByteSpan(*this));
|
||||
}
|
||||
|
||||
/** Compute total memory usage of this object (own memory + any dynamic memory). */
|
||||
@ -392,11 +366,11 @@ class AutoFile
|
||||
{
|
||||
protected:
|
||||
std::FILE* m_file;
|
||||
std::vector<std::byte> m_xor;
|
||||
Obfuscation m_obfuscation;
|
||||
std::optional<int64_t> m_position;
|
||||
|
||||
public:
|
||||
explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={});
|
||||
explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = 0);
|
||||
|
||||
~AutoFile() { fclose(); }
|
||||
|
||||
@ -428,7 +402,7 @@ public:
|
||||
bool IsNull() const { return m_file == nullptr; }
|
||||
|
||||
/** Continue with a different XOR key */
|
||||
void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; }
|
||||
void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; }
|
||||
|
||||
/** Implementation detail, only used internally. */
|
||||
std::size_t detail_fread(Span<std::byte> dst);
|
||||
@ -451,6 +425,7 @@ public:
|
||||
void read(Span<std::byte> dst);
|
||||
void ignore(size_t nSize);
|
||||
void write(Span<const std::byte> src);
|
||||
void write_large(Span<std::byte> src); // Note that src will be mutated
|
||||
|
||||
template <typename T>
|
||||
AutoFile& operator<<(const T& obj)
|
||||
|
||||
@ -14,16 +14,6 @@
|
||||
|
||||
using util::ToString;
|
||||
|
||||
// Test if a string consists entirely of null characters
|
||||
static bool is_null_key(const std::vector<unsigned char>& key) {
|
||||
bool isnull = true;
|
||||
|
||||
for (unsigned int i = 0; i < key.size(); i++)
|
||||
isnull &= (key[i] == '\x00');
|
||||
|
||||
return isnull;
|
||||
}
|
||||
|
||||
BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup)
|
||||
|
||||
BOOST_AUTO_TEST_CASE(dbwrapper)
|
||||
@ -37,7 +27,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
|
||||
uint256 res;
|
||||
|
||||
// Ensure that we're doing real obfuscation when obfuscate=true
|
||||
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
|
||||
BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
|
||||
|
||||
BOOST_CHECK(dbw.Write(key, in));
|
||||
BOOST_CHECK(dbw.Read(key, res));
|
||||
@ -57,7 +47,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data)
|
||||
bool res_bool;
|
||||
|
||||
// Ensure that we're doing real obfuscation when obfuscate=true
|
||||
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
|
||||
BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
|
||||
|
||||
//Simulate block raw data - "b + block hash"
|
||||
std::string key_block = "b" + m_rng.rand256().ToString();
|
||||
@ -232,7 +222,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
|
||||
BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
|
||||
|
||||
BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
|
||||
BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string
|
||||
BOOST_CHECK(!dbwrapper_private::GetObfuscation(odbw));
|
||||
|
||||
uint256 in2 = m_rng.rand256();
|
||||
uint256 res3;
|
||||
@ -269,7 +259,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex)
|
||||
// Check that the key/val we wrote with unobfuscated wrapper doesn't exist
|
||||
uint256 res2;
|
||||
BOOST_CHECK(!odbw.Read(key, res2));
|
||||
BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw)));
|
||||
BOOST_CHECK(dbwrapper_private::GetObfuscation(odbw));
|
||||
|
||||
uint256 in2 = m_rng.rand256();
|
||||
uint256 res3;
|
||||
|
||||
@ -20,7 +20,7 @@ FUZZ_TARGET(autofile)
|
||||
FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider};
|
||||
AutoFile auto_file{
|
||||
fuzzed_file_provider.open(),
|
||||
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
|
||||
fuzzed_data_provider.ConsumeIntegral<uint64_t>()
|
||||
};
|
||||
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100)
|
||||
{
|
||||
|
||||
@ -22,7 +22,7 @@ FUZZ_TARGET(buffered_file)
|
||||
std::optional<BufferedFile> opt_buffered_file;
|
||||
AutoFile fuzzed_file{
|
||||
fuzzed_file_provider.open(),
|
||||
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
|
||||
fuzzed_data_provider.ConsumeIntegral<uint64_t>()
|
||||
};
|
||||
try {
|
||||
auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096);
|
||||
|
||||
@ -14,16 +14,121 @@ using namespace std::string_literals;
|
||||
|
||||
BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
|
||||
|
||||
BOOST_AUTO_TEST_CASE(obfuscation_constructors)
|
||||
{
|
||||
constexpr uint64_t test_key = 0x0123456789ABCDEF;
|
||||
|
||||
// Direct uint64_t constructor
|
||||
const Obfuscation obf1{test_key};
|
||||
BOOST_CHECK_EQUAL(obf1.Key(), test_key);
|
||||
|
||||
// Span constructor
|
||||
std::array<std::byte, Obfuscation::SIZE_BYTES> key_bytes{};
|
||||
std::memcpy(key_bytes.data(), &test_key, Obfuscation::SIZE_BYTES);
|
||||
const Obfuscation obf2{Span{key_bytes}};
|
||||
BOOST_CHECK_EQUAL(obf2.Key(), test_key);
|
||||
|
||||
// std::array<std:byte> constructor
|
||||
const Obfuscation obf3{key_bytes};
|
||||
BOOST_CHECK_EQUAL(obf3.Key(), test_key);
|
||||
|
||||
// std::vector<uint8_t> constructor
|
||||
std::vector<uint8_t> uchar_key(Obfuscation::SIZE_BYTES);
|
||||
std::memcpy(uchar_key.data(), &test_key, uchar_key.size());
|
||||
const Obfuscation obf4{uchar_key};
|
||||
BOOST_CHECK_EQUAL(obf4.Key(), test_key);
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(obfuscation_serialize)
|
||||
{
|
||||
const Obfuscation original{0xDEADBEEF};
|
||||
|
||||
// Serialize
|
||||
DataStream ds;
|
||||
ds << original;
|
||||
|
||||
BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::SIZE_BYTES); // serialized as a vector
|
||||
|
||||
// Deserialize
|
||||
Obfuscation recovered{0};
|
||||
ds >> recovered;
|
||||
|
||||
BOOST_CHECK_EQUAL(recovered.Key(), original.Key());
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(obfuscation_empty)
|
||||
{
|
||||
const Obfuscation null_obf{0};
|
||||
BOOST_CHECK(!null_obf);
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(xor_bytes_reference)
|
||||
{
|
||||
auto expected_xor{[](std::span<std::byte> write, const std::span<const std::byte> key, size_t key_offset) {
|
||||
for (auto& b : write) {
|
||||
b ^= key[key_offset++ % key.size()];
|
||||
}
|
||||
}};
|
||||
|
||||
FastRandomContext rng{/*fDeterministic=*/false};
|
||||
for (size_t test{0}; test < 100; ++test) {
|
||||
const size_t write_size{1 + rng.randrange(100U)};
|
||||
const size_t key_offset{rng.randrange(3 * 8U)}; // Should wrap around
|
||||
|
||||
|
||||
const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
|
||||
const Obfuscation obfuscation{key_bytes};
|
||||
std::vector expected{rng.randbytes<std::byte>(write_size)};
|
||||
std::vector actual{expected};
|
||||
|
||||
expected_xor(expected, key_bytes, key_offset);
|
||||
obfuscation(actual, key_offset);
|
||||
|
||||
BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end());
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
|
||||
{
|
||||
auto apply_random_xor_chunks{[](std::span<std::byte> write, const Obfuscation& obfuscation, FastRandomContext& rng) {
|
||||
for (size_t offset{0}; offset < write.size();) {
|
||||
const size_t chunk_size{1 + rng.randrange(write.size() - offset)};
|
||||
obfuscation(write.subspan(offset, chunk_size), offset);
|
||||
offset += chunk_size;
|
||||
}
|
||||
}};
|
||||
|
||||
FastRandomContext rng{/*fDeterministic=*/false};
|
||||
for (size_t test{0}; test < 100; ++test) {
|
||||
const size_t write_size{1 + rng.randrange(100U)};
|
||||
const std::vector original{rng.randbytes<std::byte>(write_size)};
|
||||
std::vector roundtrip{original};
|
||||
|
||||
const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
|
||||
const Obfuscation obfuscation{key_bytes};
|
||||
apply_random_xor_chunks(roundtrip, obfuscation, rng);
|
||||
|
||||
const bool all_zero = !obfuscation || (HexStr(key_bytes).find_first_not_of('0') >= write_size * 2);
|
||||
BOOST_CHECK_EQUAL(original != roundtrip, !all_zero);
|
||||
|
||||
apply_random_xor_chunks(roundtrip, obfuscation, rng);
|
||||
BOOST_CHECK(original == roundtrip);
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(xor_file)
|
||||
{
|
||||
fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
|
||||
auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
|
||||
const std::vector<uint8_t> test1{1, 2, 3};
|
||||
const std::vector<uint8_t> test2{4, 5};
|
||||
const std::vector<std::byte> xor_pat{std::byte{0xff}, std::byte{0x00}};
|
||||
constexpr std::array xor_pat{std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}};
|
||||
uint64_t xor_key;
|
||||
std::memcpy(&xor_key, xor_pat.data(), sizeof xor_key);
|
||||
|
||||
{
|
||||
// Check errors for missing file
|
||||
AutoFile xor_file{raw_file("rb"), xor_pat};
|
||||
AutoFile xor_file{raw_file("rb"), xor_key};
|
||||
BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"});
|
||||
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"});
|
||||
BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"});
|
||||
@ -35,7 +140,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
|
||||
#else
|
||||
const char* mode = "wbx";
|
||||
#endif
|
||||
AutoFile xor_file{raw_file(mode), xor_pat};
|
||||
AutoFile xor_file{raw_file(mode), xor_key};
|
||||
xor_file << test1 << test2;
|
||||
}
|
||||
{
|
||||
@ -48,7 +153,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
|
||||
BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
|
||||
}
|
||||
{
|
||||
AutoFile xor_file{raw_file("rb"), xor_pat};
|
||||
AutoFile xor_file{raw_file("rb"), xor_key};
|
||||
std::vector<std::byte> read1, read2;
|
||||
xor_file >> read1 >> read2;
|
||||
BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
|
||||
@ -57,7 +162,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
|
||||
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
|
||||
}
|
||||
{
|
||||
AutoFile xor_file{raw_file("rb"), xor_pat};
|
||||
AutoFile xor_file{raw_file("rb"), xor_key};
|
||||
std::vector<std::byte> read2;
|
||||
// Check that ignore works
|
||||
xor_file.ignore(4);
|
||||
@ -73,7 +178,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer)
|
||||
{
|
||||
unsigned char a(1);
|
||||
unsigned char b(2);
|
||||
unsigned char bytes[] = { 3, 4, 5, 6 };
|
||||
unsigned char bytes[] = {3, 4, 5, 6};
|
||||
std::vector<unsigned char> vch;
|
||||
|
||||
// Each test runs twice. Serializing a second time at the same starting
|
||||
@ -225,29 +330,30 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
|
||||
// Degenerate case
|
||||
{
|
||||
DataStream ds{in};
|
||||
ds.Xor({0x00, 0x00});
|
||||
Obfuscation{0}(ds);
|
||||
BOOST_CHECK_EQUAL(""s, ds.str());
|
||||
}
|
||||
|
||||
in.push_back(std::byte{0x0f});
|
||||
in.push_back(std::byte{0xf0});
|
||||
|
||||
// Single character key
|
||||
{
|
||||
const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}}};
|
||||
|
||||
DataStream ds{in};
|
||||
ds.Xor({0xff});
|
||||
obfuscation(ds);
|
||||
BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
|
||||
}
|
||||
|
||||
// Multi character key
|
||||
|
||||
in.clear();
|
||||
in.push_back(std::byte{0xf0});
|
||||
in.push_back(std::byte{0x0f});
|
||||
|
||||
{
|
||||
const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}}};
|
||||
|
||||
DataStream ds{in};
|
||||
ds.Xor({0xff, 0x0f});
|
||||
obfuscation(ds);
|
||||
BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
|
||||
}
|
||||
}
|
||||
@ -270,7 +376,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
|
||||
BOOST_CHECK(false);
|
||||
} catch (const std::exception& e) {
|
||||
BOOST_CHECK(strstr(e.what(),
|
||||
"Rewind limit must be less than buffer size") != nullptr);
|
||||
"Rewind limit must be less than buffer size") != nullptr);
|
||||
}
|
||||
|
||||
// The buffer is 25 bytes, allow rewinding 10 bytes.
|
||||
@ -359,7 +465,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
|
||||
BOOST_CHECK(false);
|
||||
} catch (const std::exception& e) {
|
||||
BOOST_CHECK(strstr(e.what(),
|
||||
"BufferedFile::Fill: end of file") != nullptr);
|
||||
"BufferedFile::Fill: end of file") != nullptr);
|
||||
}
|
||||
// Attempting to read beyond the end sets the EOF indicator.
|
||||
BOOST_CHECK(bf.eof());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user