From afc534df9adbf5599b286b5dc3531a4b9ac2d056 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Thu, 13 Jul 2023 21:17:45 +0200 Subject: [PATCH 01/16] refactor: Wrap DestroyDB in dbwrapper helper Wrap leveldb::DestroyDB in a helper function without exposing leveldb-specifics. Also, add missing optional include. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 5 +++++ src/dbwrapper.h | 9 ++++----- src/validation.cpp | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 2aade14ef49..05d24f75070 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -27,6 +27,11 @@ #include #include +bool DestroyDB(const std::string& path_str) +{ + return leveldb::DestroyDB(path_str, {}).ok(); +} + class CBitcoinLevelDBLogger : public leveldb::Logger { public: // This code is adapted from posix_logger.h, which is why it is using vsprintf. diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 4ae2106211a..478b73d56f1 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -64,10 +65,6 @@ public: class CDBWrapper; -namespace dbwrapper { - using leveldb::DestroyDB; -} - /** These should be considered an implementation detail of the specific database. */ namespace dbwrapper_private { @@ -82,7 +79,9 @@ void HandleError(const leveldb::Status& status); */ const std::vector& GetObfuscateKey(const CDBWrapper &w); -}; +}; // namespace dbwrapper_private + +bool DestroyDB(const std::string& path_str); /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch diff --git a/src/validation.cpp b/src/validation.cpp index e6def01db55..e4b5381f001 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5027,7 +5027,7 @@ static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) // We have to destruct before this call leveldb::DB in order to release the db // lock, otherwise `DestroyDB` will fail. See `leveldb::~DBImpl()`. - const bool destroyed = dbwrapper::DestroyDB(path_str, {}).ok(); + const bool destroyed = DestroyDB(path_str); if (!destroyed) { LogPrintf("error: leveldb DestroyDB call failed on %s\n", path_str); From 532ee812a499e13b123af6b8415d8de1f3804f0f Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Thu, 13 Jul 2023 21:38:46 +0200 Subject: [PATCH 02/16] refactor: Split dbwrapper CDBBatch::Write implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 20 ++++++++++++++++++++ src/dbwrapper.h | 19 ++++--------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 05d24f75070..b909ce75b5a 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include #include @@ -23,7 +25,9 @@ #include #include #include +#include #include +#include #include #include @@ -132,6 +136,22 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } +void CDBBatch::WriteImpl(Span ssKey, CDataStream& ssValue) +{ + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); + leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); + batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); +} + CDBWrapper::CDBWrapper(const DBParams& params) : m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only} { diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 478b73d56f1..81c75f651ec 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -97,6 +97,8 @@ private: size_t size_estimate{0}; + void WriteImpl(Span ssKey, CDataStream& ssValue); + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to @@ -113,23 +115,10 @@ public: void Write(const K& key, const V& value) { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); - ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); + ssKey << key; ssValue << value; - ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); - leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); - - batch.Put(slKey, slValue); - // LevelDB serializes writes as: - // - byte: header - // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) - // - byte[]: key - // - varint: value length - // - byte[]: value - // The formula below assumes the key and value are both less than 16k. - size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); + WriteImpl(ssKey, ssValue); ssKey.clear(); ssValue.clear(); } From b9870c920dc475ec759eaf7339ea42aecba92138 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Thu, 13 Jul 2023 21:46:25 +0200 Subject: [PATCH 03/16] refactor: Split dbwrapper CDBatch::Erase implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 12 ++++++++++++ src/dbwrapper.h | 11 ++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index b909ce75b5a..b74c645b8f9 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -152,6 +152,18 @@ void CDBBatch::WriteImpl(Span ssKey, CDataStream& ssValue) size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); } +void CDBBatch::EraseImpl(Span ssKey) +{ + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); +} + CDBWrapper::CDBWrapper(const DBParams& params) : m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only} { diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 81c75f651ec..61f5003db74 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -98,6 +98,7 @@ private: size_t size_estimate{0}; void WriteImpl(Span ssKey, CDataStream& ssValue); + void EraseImpl(Span ssKey); public: /** @@ -128,15 +129,7 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - - batch.Delete(slKey); - // LevelDB serializes erases as: - // - byte: header - // - varint: key length - // - byte[]: key - // The formula below assumes the key is less than 16kB. - size_estimate += 2 + (slKey.size() > 127) + slKey.size(); + EraseImpl(ssKey); ssKey.clear(); } From ea8135de7e617259cda3fc7b1c8e7569d454fd57 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Thu, 13 Jul 2023 21:55:07 +0200 Subject: [PATCH 04/16] refactor: Pimpl leveldb::batch for CDBBatch Hide the leveldb::WriteBatch member variable with a pimpl in order not to expose it directly in the header. Also move CDBBatch::Clear to the dbwrapper implementation to use the new impl_batch. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 24 +++++++++++++++++++++--- src/dbwrapper.h | 16 +++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index b74c645b8f9..ff2bc8b4e88 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,8 +4,10 @@ #include +#include #include #include +#include #include #include #include @@ -136,12 +138,28 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } +struct CDBBatch::WriteBatchImpl { + leveldb::WriteBatch batch; +}; + +CDBBatch::CDBBatch(const CDBWrapper& _parent) : parent(_parent), + m_impl_batch{std::make_unique()}, + ssValue(SER_DISK, CLIENT_VERSION){}; + +CDBBatch::~CDBBatch() = default; + +void CDBBatch::Clear() +{ + m_impl_batch->batch.Clear(); + size_estimate = 0; +} + void CDBBatch::WriteImpl(Span ssKey, CDataStream& ssValue) { leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); - batch.Put(slKey, slValue); + m_impl_batch->batch.Put(slKey, slValue); // LevelDB serializes writes as: // - byte: header // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) @@ -155,7 +173,7 @@ void CDBBatch::WriteImpl(Span ssKey, CDataStream& ssValue) void CDBBatch::EraseImpl(Span ssKey) { leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - batch.Delete(slKey); + m_impl_batch->batch.Delete(slKey); // LevelDB serializes erases as: // - byte: header // - varint: key length @@ -241,7 +259,7 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) if (log_memory) { mem_before = DynamicMemoryUsage() / 1024.0 / 1024; } - leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); + leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.m_impl_batch->batch); dbwrapper_private::HandleError(status); if (log_memory) { double mem_after = DynamicMemoryUsage() / 1024.0 / 1024; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 61f5003db74..9016111209d 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -90,7 +90,9 @@ class CDBBatch private: const CDBWrapper &parent; - leveldb::WriteBatch batch; + + struct WriteBatchImpl; + const std::unique_ptr m_impl_batch; DataStream ssKey{}; CDataStream ssValue; @@ -104,13 +106,9 @@ public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - explicit CDBBatch(const CDBWrapper& _parent) : parent(_parent), ssValue(SER_DISK, CLIENT_VERSION){}; - - void Clear() - { - batch.Clear(); - size_estimate = 0; - } + explicit CDBBatch(const CDBWrapper& _parent); + ~CDBBatch(); + void Clear(); template void Write(const K& key, const V& value) From d7437908cdf242626263ba9d5541addcddadc594 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 28 Jul 2023 22:53:50 +0200 Subject: [PATCH 05/16] refactor: Split dbwrapper CDBIterator::Seek implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 6 ++++++ src/dbwrapper.h | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index ff2bc8b4e88..87d46f31925 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -306,6 +306,12 @@ bool CDBWrapper::IsEmpty() return !(it->Valid()); } +void CDBIterator::SeekImpl(Span ssKey) +{ + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + piter->Seek(slKey); +} + CDBIterator::~CDBIterator() { delete piter; } bool CDBIterator::Valid() const { return piter->Valid(); } void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 9016111209d..aa2b31b160e 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -140,6 +140,8 @@ private: const CDBWrapper &parent; leveldb::Iterator *piter; + void SeekImpl(Span ssKey); + public: /** @@ -158,8 +160,7 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - piter->Seek(slKey); + SeekImpl(ssKey); } void Next(); From b7a1ab5cb4e60230f62c94efb3a10d07c9af4883 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 14 Jul 2023 11:21:26 +0200 Subject: [PATCH 06/16] refactor: Split dbwrapper CDBIterator::GetKey implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 5 +++++ src/dbwrapper.h | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 87d46f31925..dbf1d18de2b 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -312,6 +312,11 @@ void CDBIterator::SeekImpl(Span ssKey) piter->Seek(slKey); } +Span CDBIterator::GetKeyImpl() const +{ + return MakeByteSpan(piter->key()); +} + CDBIterator::~CDBIterator() { delete piter; } bool CDBIterator::Valid() const { return piter->Valid(); } void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index aa2b31b160e..9aadeef76db 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -141,6 +141,7 @@ private: leveldb::Iterator *piter; void SeekImpl(Span ssKey); + Span GetKeyImpl() const; public: @@ -166,9 +167,8 @@ public: void Next(); template bool GetKey(K& key) { - leveldb::Slice slKey = piter->key(); try { - DataStream ssKey{MakeByteSpan(slKey)}; + DataStream ssKey{GetKeyImpl()}; ssKey >> key; } catch (const std::exception&) { return false; From ef941ff1281e76308c3e746e592375bec023e9e4 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 14 Jul 2023 11:30:52 +0200 Subject: [PATCH 07/16] refactor: Split dbwrapper CDBIterator::GetValue implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 5 +++++ src/dbwrapper.h | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index dbf1d18de2b..2f0b9a5ff34 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -317,6 +317,11 @@ Span CDBIterator::GetKeyImpl() const return MakeByteSpan(piter->key()); } +Span CDBIterator::GetValueImpl() const +{ + return MakeByteSpan(piter->value()); +} + CDBIterator::~CDBIterator() { delete piter; } bool CDBIterator::Valid() const { return piter->Valid(); } void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 9aadeef76db..2e6cc4d81e0 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -27,6 +26,7 @@ #include namespace leveldb { class Env; +class Iterator; } static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; @@ -142,6 +142,7 @@ private: void SeekImpl(Span ssKey); Span GetKeyImpl() const; + Span GetValueImpl() const; public: @@ -177,9 +178,8 @@ public: } template bool GetValue(V& value) { - leveldb::Slice slValue = piter->value(); try { - CDataStream ssValue{MakeByteSpan(slValue), SER_DISK, CLIENT_VERSION}; + CDataStream ssValue{GetValueImpl(), SER_DISK, CLIENT_VERSION}; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { From e4af2408f2ac59788567b6fc8cb3a68fc43da9fe Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Sat, 29 Jul 2023 12:03:35 +0200 Subject: [PATCH 08/16] refactor: Pimpl leveldb::Iterator for CDBIterator Hide the leveldb::Iterator member variable with a pimpl in order not to expose it directly in the header. Also, move CDBWrapper::NewIterator to the dbwrapper implementation to use the pimpl for CDBIterator initialziation. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 28 +++++++++++++++++++++------- src/dbwrapper.h | 14 ++++++-------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 2f0b9a5ff34..c0a2895a587 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -32,6 +32,7 @@ #include #include #include +#include bool DestroyDB(const std::string& path_str) { @@ -306,26 +307,39 @@ bool CDBWrapper::IsEmpty() return !(it->Valid()); } +struct CDBIterator::IteratorImpl { + const std::unique_ptr iter; + + explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {} +}; + +CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr _piter) : parent(_parent), m_impl_iter(std::move(_piter)) {} + +CDBIterator* CDBWrapper::NewIterator() +{ + return new CDBIterator{*this, std::make_unique(pdb->NewIterator(iteroptions))}; +} + void CDBIterator::SeekImpl(Span ssKey) { leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - piter->Seek(slKey); + m_impl_iter->iter->Seek(slKey); } Span CDBIterator::GetKeyImpl() const { - return MakeByteSpan(piter->key()); + return MakeByteSpan(m_impl_iter->iter->key()); } Span CDBIterator::GetValueImpl() const { - return MakeByteSpan(piter->value()); + return MakeByteSpan(m_impl_iter->iter->value()); } -CDBIterator::~CDBIterator() { delete piter; } -bool CDBIterator::Valid() const { return piter->Valid(); } -void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CDBIterator::Next() { piter->Next(); } +CDBIterator::~CDBIterator() = default; +bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); } +void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); } +void CDBIterator::Next() { m_impl_iter->iter->Next(); } namespace dbwrapper_private { diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 2e6cc4d81e0..9bcb79c169e 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -26,7 +26,6 @@ #include namespace leveldb { class Env; -class Iterator; } static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; @@ -136,9 +135,12 @@ public: class CDBIterator { +public: + struct IteratorImpl; + private: const CDBWrapper &parent; - leveldb::Iterator *piter; + const std::unique_ptr m_impl_iter; void SeekImpl(Span ssKey); Span GetKeyImpl() const; @@ -150,8 +152,7 @@ public: * @param[in] _parent Parent CDBWrapper instance. * @param[in] _piter The original leveldb iterator. */ - CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) : - parent(_parent), piter(_piter) { }; + CDBIterator(const CDBWrapper& _parent, std::unique_ptr _piter); ~CDBIterator(); bool Valid() const; @@ -315,10 +316,7 @@ public: // Get an estimate of LevelDB memory usage (in bytes). size_t DynamicMemoryUsage() const; - CDBIterator *NewIterator() - { - return new CDBIterator(*this, pdb->NewIterator(iteroptions)); - } + CDBIterator* NewIterator(); /** * Return true if the database managed by this class contains no entries. From 84058e0eed9c05bc30984b39131e88ad1425628f Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 14 Jul 2023 11:46:13 +0200 Subject: [PATCH 09/16] refactor: Split dbwrapper CDBWrapper::Read implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 15 ++++++++++++++- src/dbwrapper.h | 16 ++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index c0a2895a587..830c04c4cf5 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -300,6 +299,20 @@ std::vector CDBWrapper::CreateObfuscateKey() const return ret; } +std::optional CDBWrapper::ReadImpl(Span ssKey) const +{ + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + std::string strValue; + leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); + if (!status.ok()) { + if (status.IsNotFound()) + return std::nullopt; + LogPrintf("LevelDB read failure: %s\n", status.ToString()); + dbwrapper_private::HandleError(status); + } + return strValue; +} + bool CDBWrapper::IsEmpty() { std::unique_ptr it(NewIterator()); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 9bcb79c169e..7e5ffe10620 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -235,6 +235,8 @@ private: //! whether or not the database resides in memory bool m_is_memory; + std::optional ReadImpl(Span ssKey) const; + public: CDBWrapper(const DBParams& params); ~CDBWrapper(); @@ -248,18 +250,12 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - - std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - LogPrintf("LevelDB read failure: %s\n", status.ToString()); - dbwrapper_private::HandleError(status); + std::optional strValue{ReadImpl(ssKey)}; + if (!strValue) { + return false; } try { - CDataStream ssValue{MakeByteSpan(strValue), SER_DISK, CLIENT_VERSION}; + CDataStream ssValue{MakeByteSpan(*strValue), SER_DISK, CLIENT_VERSION}; ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { From a5c2eb57484314b04ec94523d14e0ef0c6c46d4f Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Sat, 29 Jul 2023 13:06:01 +0200 Subject: [PATCH 10/16] refactor: Fix logging.h includes These were uncovered as missing by the next commit. --- src/blockencodings.cpp | 1 + src/index/blockfilterindex.cpp | 1 + src/init.cpp | 1 + src/net_processing.cpp | 1 + src/node/interfaces.cpp | 1 + src/node/miner.cpp | 1 + src/qt/bitcoin.cpp | 3 ++- src/qt/test/apptests.cpp | 1 + src/qt/transactiondesc.cpp | 1 + src/rpc/node.cpp | 1 + src/test/util/setup_common.cpp | 1 + 11 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 9aa0a6ba204..211b4740be9 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index a860b3a94d6..cc7d6687b8e 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/init.cpp b/src/init.cpp index c11f100139d..685c135b745 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include diff --git a/src/net_processing.cpp b/src/net_processing.cpp index be6777d14bc..11be981cd9d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 9c98e4cf0cd..42e021fcc9e 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/src/node/miner.cpp b/src/node/miner.cpp index aa1a9a155c6..caa29918193 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8f45af9485d..865871a6d43 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -17,6 +16,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index fb8029cb650..b05009965f4 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index fa110cfbc9b..dae6a2dea97 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 38284016424..80cc458377c 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 65c657da96b..08ef890ec4c 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include From dede0eef7adb7413f62f5abd68cac8e01635ba4a Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 14 Jul 2023 11:53:24 +0200 Subject: [PATCH 11/16] refactor: Split dbwrapper CDBWrapper::Exists implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 15 +++++++++++++++ src/dbwrapper.h | 16 +++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 830c04c4cf5..88eb7e4aace 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -313,6 +313,21 @@ std::optional CDBWrapper::ReadImpl(Span ssKey) con return strValue; } +bool CDBWrapper::ExistsImpl(Span ssKey) const +{ + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + + std::string strValue; + leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); + if (!status.ok()) { + if (status.IsNotFound()) + return false; + LogPrintf("LevelDB read failure: %s\n", status.ToString()); + dbwrapper_private::HandleError(status); + } + return true; +} + bool CDBWrapper::IsEmpty() { std::unique_ptr it(NewIterator()); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 7e5ffe10620..5b9ff6ea91c 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -6,7 +6,6 @@ #define BITCOIN_DBWRAPPER_H #include -#include #include #include #include @@ -18,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -26,6 +24,7 @@ #include namespace leveldb { class Env; +class Status; } static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; @@ -236,6 +235,7 @@ private: bool m_is_memory; std::optional ReadImpl(Span ssKey) const; + bool ExistsImpl(Span ssKey) const; public: CDBWrapper(const DBParams& params); @@ -286,17 +286,7 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); - - std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); - if (!status.ok()) { - if (status.IsNotFound()) - return false; - LogPrintf("LevelDB read failure: %s\n", status.ToString()); - dbwrapper_private::HandleError(status); - } - return true; + return ExistsImpl(ssKey); } template From 586448888b72f7c87db4dcd30fc4e4044afae13b Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Sat, 29 Jul 2023 10:04:50 +0200 Subject: [PATCH 12/16] refactor: Move HandleError to dbwrapper implementation Make it a static function in dbwrapper.cpp, since it is not used elsewhere and when left in the header, would expose a leveldb type. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 32 +++++++++++++++++--------------- src/dbwrapper.h | 5 ----- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 88eb7e4aace..fe0a2e66f2f 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -38,6 +38,18 @@ bool DestroyDB(const std::string& path_str) return leveldb::DestroyDB(path_str, {}).ok(); } +/** Handle database error by throwing dbwrapper_error exception. + */ +static void HandleError(const leveldb::Status& status) +{ + if (status.ok()) + return; + const std::string errmsg = "Fatal LevelDB error: " + status.ToString(); + LogPrintf("%s\n", errmsg); + LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n"); + throw dbwrapper_error(errmsg); +} + class CBitcoinLevelDBLogger : public leveldb::Logger { public: // This code is adapted from posix_logger.h, which is why it is using vsprintf. @@ -199,7 +211,7 @@ CDBWrapper::CDBWrapper(const DBParams& params) if (params.wipe_data) { LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path)); leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), options); - dbwrapper_private::HandleError(result); + HandleError(result); } TryCreateDirectories(params.path); LogPrintf("Opening LevelDB in %s\n", fs::PathToString(params.path)); @@ -209,7 +221,7 @@ CDBWrapper::CDBWrapper(const DBParams& params) // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW // (see env_posix.cc and env_windows.cc). leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(params.path), &pdb); - dbwrapper_private::HandleError(status); + HandleError(status); LogPrintf("Opened LevelDB successfully\n"); if (params.options.force_compact) { @@ -260,7 +272,7 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) mem_before = DynamicMemoryUsage() / 1024.0 / 1024; } leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.m_impl_batch->batch); - dbwrapper_private::HandleError(status); + HandleError(status); if (log_memory) { double mem_after = DynamicMemoryUsage() / 1024.0 / 1024; LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n", @@ -308,7 +320,7 @@ std::optional CDBWrapper::ReadImpl(Span ssKey) con if (status.IsNotFound()) return std::nullopt; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - dbwrapper_private::HandleError(status); + HandleError(status); } return strValue; } @@ -323,7 +335,7 @@ bool CDBWrapper::ExistsImpl(Span ssKey) const if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - dbwrapper_private::HandleError(status); + HandleError(status); } return true; } @@ -371,16 +383,6 @@ void CDBIterator::Next() { m_impl_iter->iter->Next(); } namespace dbwrapper_private { -void HandleError(const leveldb::Status& status) -{ - if (status.ok()) - return; - const std::string errmsg = "Fatal LevelDB error: " + status.ToString(); - LogPrintf("%s\n", errmsg); - LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n"); - throw dbwrapper_error(errmsg); -} - const std::vector& GetObfuscateKey(const CDBWrapper &w) { return w.obfuscate_key; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5b9ff6ea91c..9756207aec7 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -24,7 +24,6 @@ #include namespace leveldb { class Env; -class Status; } static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; @@ -67,10 +66,6 @@ class CDBWrapper; */ namespace dbwrapper_private { -/** Handle database error by throwing dbwrapper_error exception. - */ -void HandleError(const leveldb::Status& status); - /** Work around circular dependency, as well as for testing in dbwrapper_tests. * Database obfuscation should be considered an implementation detail of the * specific database. From c534a615e93452a5f509aaf5f68c600391a98d6a Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 14 Jul 2023 12:04:41 +0200 Subject: [PATCH 13/16] refactor: Split dbwrapper CDBWrapper::EstimateSize implementation Keep the generic serialization in the header, while moving leveldb-specifics to the implementation file. Since CharCast is no longer needed in the header, move it to the implementation file. The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 12 ++++++++++++ src/dbwrapper.h | 14 +++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index fe0a2e66f2f..58fd47ce1ca 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -33,6 +33,8 @@ #include #include +static auto CharCast(const std::byte* data) { return reinterpret_cast(data); } + bool DestroyDB(const std::string& path_str) { return leveldb::DestroyDB(path_str, {}).ok(); @@ -340,6 +342,16 @@ bool CDBWrapper::ExistsImpl(Span ssKey) const return true; } +size_t CDBWrapper::EstimateSizeImpl(Span ssKey1, Span ssKey2) const +{ + leveldb::Slice slKey1(CharCast(ssKey1.data()), ssKey1.size()); + leveldb::Slice slKey2(CharCast(ssKey2.data()), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; +} + bool CDBWrapper::IsEmpty() { std::unique_ptr it(NewIterator()); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 9756207aec7..fa7b6397cba 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -12,17 +12,15 @@ #include #include -#include #include -#include #include -#include #include #include #include #include #include namespace leveldb { +class DB; class Env; } @@ -52,8 +50,6 @@ struct DBParams { DBOptions options{}; }; -inline auto CharCast(const std::byte* data) { return reinterpret_cast(data); } - class dbwrapper_error : public std::runtime_error { public: @@ -231,6 +227,7 @@ private: std::optional ReadImpl(Span ssKey) const; bool ExistsImpl(Span ssKey) const; + size_t EstimateSizeImpl(Span ssKey1, Span ssKey2) const; public: CDBWrapper(const DBParams& params); @@ -312,12 +309,7 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(CharCast(ssKey1.data()), ssKey1.size()); - leveldb::Slice slKey2(CharCast(ssKey2.data()), ssKey2.size()); - uint64_t size = 0; - leveldb::Range range(slKey1, slKey2); - pdb->GetApproximateSizes(&range, 1, &size); - return size; + return EstimateSizeImpl(ssKey1, ssKey2); } }; From c95b37d641b1eed4a62d55ca5342a6ed8c7a1ce7 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Fri, 14 Jul 2023 12:31:20 +0200 Subject: [PATCH 14/16] refactor: Move CDBWrapper leveldb members to their own context struct The context of this commit is an effort to decouple the dbwrapper header file from leveldb includes. To this end, the includes are moved to the dbwrapper implementation file. This is done as part of the kernel project to reduce the number of required includes for users of the kernel. --- src/dbwrapper.cpp | 84 ++++++++++++++++++++++++++++++----------------- src/dbwrapper.h | 32 ++++-------------- 2 files changed, 61 insertions(+), 55 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 58fd47ce1ca..b8ee852bb52 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -196,23 +196,46 @@ void CDBBatch::EraseImpl(Span ssKey) size_estimate += 2 + (slKey.size() > 127) + slKey.size(); } +struct LevelDBContext { + //! custom environment this database is using (may be nullptr in case of default environment) + leveldb::Env* penv; + + //! database options used + leveldb::Options options; + + //! options used when reading from the database + leveldb::ReadOptions readoptions; + + //! options used when iterating over values of the database + leveldb::ReadOptions iteroptions; + + //! options used when writing to the database + leveldb::WriteOptions writeoptions; + + //! options used when sync writing to the database + leveldb::WriteOptions syncoptions; + + //! the database itself + leveldb::DB* pdb; +}; + CDBWrapper::CDBWrapper(const DBParams& params) - : m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only} + : m_db_context{std::make_unique()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only} { - penv = nullptr; - readoptions.verify_checksums = true; - iteroptions.verify_checksums = true; - iteroptions.fill_cache = false; - syncoptions.sync = true; - options = GetOptions(params.cache_bytes); - options.create_if_missing = true; + DBContext().penv = nullptr; + DBContext().readoptions.verify_checksums = true; + DBContext().iteroptions.verify_checksums = true; + DBContext().iteroptions.fill_cache = false; + DBContext().syncoptions.sync = true; + DBContext().options = GetOptions(params.cache_bytes); + DBContext().options.create_if_missing = true; if (params.memory_only) { - penv = leveldb::NewMemEnv(leveldb::Env::Default()); - options.env = penv; + DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default()); + DBContext().options.env = DBContext().penv; } else { if (params.wipe_data) { LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path)); - leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), options); + leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options); HandleError(result); } TryCreateDirectories(params.path); @@ -222,13 +245,13 @@ CDBWrapper::CDBWrapper(const DBParams& params) // because on POSIX leveldb passes the byte string directly to ::open(), and // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW // (see env_posix.cc and env_windows.cc). - leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(params.path), &pdb); + leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb); HandleError(status); LogPrintf("Opened LevelDB successfully\n"); if (params.options.force_compact) { LogPrintf("Starting database compaction of %s\n", fs::PathToString(params.path)); - pdb->CompactRange(nullptr, nullptr); + DBContext().pdb->CompactRange(nullptr, nullptr); LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path)); } @@ -254,16 +277,16 @@ CDBWrapper::CDBWrapper(const DBParams& params) CDBWrapper::~CDBWrapper() { - delete pdb; - pdb = nullptr; - delete options.filter_policy; - options.filter_policy = nullptr; - delete options.info_log; - options.info_log = nullptr; - delete options.block_cache; - options.block_cache = nullptr; - delete penv; - options.env = nullptr; + delete DBContext().pdb; + DBContext().pdb = nullptr; + delete DBContext().options.filter_policy; + DBContext().options.filter_policy = nullptr; + delete DBContext().options.info_log; + DBContext().options.info_log = nullptr; + delete DBContext().options.block_cache; + DBContext().options.block_cache = nullptr; + delete DBContext().penv; + DBContext().options.env = nullptr; } bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) @@ -273,7 +296,7 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) if (log_memory) { mem_before = DynamicMemoryUsage() / 1024.0 / 1024; } - leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.m_impl_batch->batch); + leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch); HandleError(status); if (log_memory) { double mem_after = DynamicMemoryUsage() / 1024.0 / 1024; @@ -287,7 +310,7 @@ size_t CDBWrapper::DynamicMemoryUsage() const { std::string memory; std::optional parsed; - if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral(memory))) { + if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral(memory))) { LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n"); return 0; } @@ -317,7 +340,7 @@ std::optional CDBWrapper::ReadImpl(Span ssKey) con { leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); + leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue); if (!status.ok()) { if (status.IsNotFound()) return std::nullopt; @@ -332,7 +355,7 @@ bool CDBWrapper::ExistsImpl(Span ssKey) const leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); std::string strValue; - leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); + leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue); if (!status.ok()) { if (status.IsNotFound()) return false; @@ -348,7 +371,7 @@ size_t CDBWrapper::EstimateSizeImpl(Span ssKey1, SpanGetApproximateSizes(&range, 1, &size); + DBContext().pdb->GetApproximateSizes(&range, 1, &size); return size; } @@ -365,11 +388,12 @@ struct CDBIterator::IteratorImpl { explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {} }; -CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr _piter) : parent(_parent), m_impl_iter(std::move(_piter)) {} +CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr _piter) : parent(_parent), + m_impl_iter(std::move(_piter)) {} CDBIterator* CDBWrapper::NewIterator() { - return new CDBIterator{*this, std::make_unique(pdb->NewIterator(iteroptions))}; + return new CDBIterator{*this, std::make_unique(DBContext().pdb->NewIterator(DBContext().iteroptions))}; } void CDBIterator::SeekImpl(Span ssKey) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index fa7b6397cba..66170b5efa2 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -5,24 +5,21 @@ #ifndef BITCOIN_DBWRAPPER_H #define BITCOIN_DBWRAPPER_H +#include #include #include #include #include +#include #include #include #include -#include #include #include #include #include #include -namespace leveldb { -class DB; -class Env; -} static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; @@ -180,30 +177,14 @@ public: } }; +struct LevelDBContext; + class CDBWrapper { friend const std::vector& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w); private: - //! custom environment this database is using (may be nullptr in case of default environment) - leveldb::Env* penv; - - //! database options used - leveldb::Options options; - - //! options used when reading from the database - leveldb::ReadOptions readoptions; - - //! options used when iterating over values of the database - leveldb::ReadOptions iteroptions; - - //! options used when writing to the database - leveldb::WriteOptions writeoptions; - - //! options used when sync writing to the database - leveldb::WriteOptions syncoptions; - - //! the database itself - leveldb::DB* pdb; + //! holds all leveldb-specific fields of this class + std::unique_ptr m_db_context; //! the name of this database std::string m_name; @@ -228,6 +209,7 @@ private: std::optional ReadImpl(Span ssKey) const; bool ExistsImpl(Span ssKey) const; size_t EstimateSizeImpl(Span ssKey1, Span ssKey2) const; + auto& DBContext() const LIFETIMEBOUND { return *Assert(m_db_context); } public: CDBWrapper(const DBParams& params); From be8f159ac59b9e700cbd3314ed71ebf39bd5b67a Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Sun, 30 Jul 2023 09:55:43 +0200 Subject: [PATCH 15/16] build: Remove leveldb from BITCOIN_INCLUDES Since leveldb is no longer in our header tree, move its include flags to whereever dbwrapper.cpp is built. --- src/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index dfea7146aad..1238bbec12b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,7 +24,7 @@ check_PROGRAMS = TESTS = BENCHMARKS = -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -370,7 +370,7 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # node # -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ addrdb.cpp \ From d8f1222ac50f089a0af29eaf8ce0555bad8366ef Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Thu, 3 Aug 2023 16:07:59 +0200 Subject: [PATCH 16/16] refactor: Correct dbwrapper key naming The ss- prefix should connotate a DataStream variable. Now that these variables are byte spans, drop the prefix. --- src/dbwrapper.cpp | 26 +++++++++++++------------- src/dbwrapper.h | 12 ++++++------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index b8ee852bb52..847d40eefad 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -168,9 +168,9 @@ void CDBBatch::Clear() size_estimate = 0; } -void CDBBatch::WriteImpl(Span ssKey, CDataStream& ssValue) +void CDBBatch::WriteImpl(Span key, CDataStream& ssValue) { - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + leveldb::Slice slKey(CharCast(key.data()), key.size()); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); m_impl_batch->batch.Put(slKey, slValue); @@ -184,9 +184,9 @@ void CDBBatch::WriteImpl(Span ssKey, CDataStream& ssValue) size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); } -void CDBBatch::EraseImpl(Span ssKey) +void CDBBatch::EraseImpl(Span key) { - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + leveldb::Slice slKey(CharCast(key.data()), key.size()); m_impl_batch->batch.Delete(slKey); // LevelDB serializes erases as: // - byte: header @@ -336,9 +336,9 @@ std::vector CDBWrapper::CreateObfuscateKey() const return ret; } -std::optional CDBWrapper::ReadImpl(Span ssKey) const +std::optional CDBWrapper::ReadImpl(Span key) const { - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + leveldb::Slice slKey(CharCast(key.data()), key.size()); std::string strValue; leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue); if (!status.ok()) { @@ -350,9 +350,9 @@ std::optional CDBWrapper::ReadImpl(Span ssKey) con return strValue; } -bool CDBWrapper::ExistsImpl(Span ssKey) const +bool CDBWrapper::ExistsImpl(Span key) const { - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + leveldb::Slice slKey(CharCast(key.data()), key.size()); std::string strValue; leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue); @@ -365,10 +365,10 @@ bool CDBWrapper::ExistsImpl(Span ssKey) const return true; } -size_t CDBWrapper::EstimateSizeImpl(Span ssKey1, Span ssKey2) const +size_t CDBWrapper::EstimateSizeImpl(Span key1, Span key2) const { - leveldb::Slice slKey1(CharCast(ssKey1.data()), ssKey1.size()); - leveldb::Slice slKey2(CharCast(ssKey2.data()), ssKey2.size()); + leveldb::Slice slKey1(CharCast(key1.data()), key1.size()); + leveldb::Slice slKey2(CharCast(key2.data()), key2.size()); uint64_t size = 0; leveldb::Range range(slKey1, slKey2); DBContext().pdb->GetApproximateSizes(&range, 1, &size); @@ -396,9 +396,9 @@ CDBIterator* CDBWrapper::NewIterator() return new CDBIterator{*this, std::make_unique(DBContext().pdb->NewIterator(DBContext().iteroptions))}; } -void CDBIterator::SeekImpl(Span ssKey) +void CDBIterator::SeekImpl(Span key) { - leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); + leveldb::Slice slKey(CharCast(key.data()), key.size()); m_impl_iter->iter->Seek(slKey); } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 66170b5efa2..eac9594aa10 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -85,8 +85,8 @@ private: size_t size_estimate{0}; - void WriteImpl(Span ssKey, CDataStream& ssValue); - void EraseImpl(Span ssKey); + void WriteImpl(Span key, CDataStream& ssValue); + void EraseImpl(Span key); public: /** @@ -129,7 +129,7 @@ private: const CDBWrapper &parent; const std::unique_ptr m_impl_iter; - void SeekImpl(Span ssKey); + void SeekImpl(Span key); Span GetKeyImpl() const; Span GetValueImpl() const; @@ -206,9 +206,9 @@ private: //! whether or not the database resides in memory bool m_is_memory; - std::optional ReadImpl(Span ssKey) const; - bool ExistsImpl(Span ssKey) const; - size_t EstimateSizeImpl(Span ssKey1, Span ssKey2) const; + std::optional ReadImpl(Span key) const; + bool ExistsImpl(Span key) const; + size_t EstimateSizeImpl(Span key1, Span key2) const; auto& DBContext() const LIFETIMEBOUND { return *Assert(m_db_context); } public: