highperfocused/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2026-03-27 16:05:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge #15741: Batch write imported stuff in importmulti

Browse Source 
0db94e55d wallet: Pass WalletBatch to CWallet::UnsetWalletFlag (João Barbosa)
6cb888b37 Apply the batch treatment to CWallet::SetAddressBook via ImportScriptPubKeys (Ben Woosley)
6154a09e0 Move some of ProcessImport into CWallet::Import* (Ben Woosley)
ccb26cf34 Batch writes for importmulti (Andrew Chow)
d6576e349 Have WalletBatch automatically flush every 1000 updates (Andrew Chow)
366fe0be0 Add AddWatchOnlyWithDB, AddKeyOriginWithDB, AddCScriptWithDB functions (Andrew Chow)

Pull request description:

  Instead of writing each item to the wallet database individually, do them in batches so that the import runs faster.

  This was tested by importing a ranged descriptor for 10,000 keys.

  Current master

  ```
  $ time src/bitcoin-cli -regtest -rpcwallet=importbig importmulti '[{"desc": "sh(wpkh([73111820/44h/1h/0h]tpubDDoT2SgEjaU5rerQpfcRDWPAcwyZ5g7xxHgVAfPwidgPDKVjm89d6jJ8AQotp35Np3m6VaysfUY1C2g68wFqUmraGbzhSsMF9YBuTGxpBaW/1/*))#3w7php47", "range": [0, 10000], "timestamp": "now", "internal": true, "keypool": false, "watchonly": true}]'
  ...

  real	7m45.29s
  ```

  This PR:

  ```
  $ time src/bitcoin-cli -regtest -rpcwallet=importbig4 importmulti '[{"desc": "pkh([73111820/44h/1h/0h]tpubDDoT2SgEjaU5rerQpfcRDWPAcwyZ5g7xxHgVAfPwidgPDKVjm89d6jJ8AQotp35Np3m6VaysfUY1C2g68wFqUmraGbzhSsMF9YBuTGxpBaW/1/*)#v65yjgmc", "range": [0, 10000], "timestamp": "now", "internal": true, "keypool": false, "watchonly": true}]'
  ...

  real	3.93s
  ```

  Fixes #15739

ACKs for commit 0db94e:
  jb55:
    utACK 0db94e5
  ariard:
    Tested ACK 0db94e5
  Empact:
    re-utACK 0db94e55dc only change is re the privacy of `UnsetWalletFlagWithDB` and `AddCScriptWithDB`.

Tree-SHA512: 3481308a64c99b6129f7bd328113dc291fe58743464628931feaebdef0e6ec770ddd5c19e4f9fbc1249a200acb04aaf62a8d914d53b0a29ac1e557576659c0cc
This commit is contained in:
MeshCollider
2019-05-29 18:54:07 +12:00
parent e78c33131b 0db94e55dc
commit ed40fbb02a
5 changed files with 163 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
src/wallet/wallet.cpp
View File

@@ -320,7 +320,7 @@ bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const C
secret.GetPrivKey(),
mapKeyMetadata[pubkey.GetID()]);
}
UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET);
UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET);
return true;
}
@@ -362,12 +362,6 @@ void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata&
m_script_metadata[script_id] = meta;
}
// Writes a keymetadata for a public key. overwrite specifies whether to overwrite an existing metadata for that key if there exists one.
bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite)
{
return WalletBatch(*database).WriteKeyMetadata(meta, pubkey, overwrite);
}
void CWallet::UpgradeKeyMetadata()
{
AssertLockHeld(cs_wallet);
@@ -376,7 +370,6 @@ void CWallet::UpgradeKeyMetadata()
}
std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database);
size_t cnt = 0;
for (auto& meta_pair : mapKeyMetadata) {
CKeyMetadata& meta = meta_pair.second;
if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin
@@ -399,10 +392,6 @@ void CWallet::UpgradeKeyMetadata()
CPubKey pubkey;
if (GetPubKey(meta_pair.first, pubkey)) {
batch->WriteKeyMetadata(meta, pubkey, true);
if (++cnt % 1000 == 0) {
// avoid creating overlarge in-memory batches in case the wallet contains large amounts of keys
batch.reset(new WalletBatch(*database));
}
}
}
}
@@ -432,11 +421,17 @@ void CWallet::UpdateTimeFirstKey(int64_t nCreateTime)
}
bool CWallet::AddCScript(const CScript& redeemScript)
{
WalletBatch batch(*database);
return AddCScriptWithDB(batch, redeemScript);
}
bool CWallet::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript)
{
if (!CCryptoKeyStore::AddCScript(redeemScript))
return false;
if (WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript)) {
UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET);
if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) {
UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET);
return true;
}
return false;
@@ -457,20 +452,32 @@ bool CWallet::LoadCScript(const CScript& redeemScript)
return CCryptoKeyStore::AddCScript(redeemScript);
}
bool CWallet::AddWatchOnly(const CScript& dest)
bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest)
{
if (!CCryptoKeyStore::AddWatchOnly(dest))
return false;
const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)];
UpdateTimeFirstKey(meta.nCreateTime);
NotifyWatchonlyChanged(true);
if (WalletBatch(*database).WriteWatchOnly(dest, meta)) {
UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET);
if (batch.WriteWatchOnly(dest, meta)) {
UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET);
return true;
}
return false;
}
bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time)
{
m_script_metadata[CScriptID(dest)].nCreateTime = create_time;
return AddWatchOnlyWithDB(batch, dest);
}
bool CWallet::AddWatchOnly(const CScript& dest)
{
WalletBatch batch(*database);
return AddWatchOnlyWithDB(batch, dest);
}
bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime)
{
m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime;
@@ -1542,10 +1549,16 @@ void CWallet::SetWalletFlag(uint64_t flags)
}
void CWallet::UnsetWalletFlag(uint64_t flag)
{
WalletBatch batch(*database);
UnsetWalletFlagWithDB(batch, flag);
}
void CWallet::UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag)
{
LOCK(cs_wallet);
m_wallet_flags &= ~flag;
if (!WalletBatch(*database).WriteWalletFlags(m_wallet_flags))
if (!batch.WriteWalletFlags(m_wallet_flags))
throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed");
}
@@ -1606,6 +1619,80 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut>
return true;
}
bool CWallet::ImportScripts(const std::set<CScript> scripts)
{
WalletBatch batch(*database);
for (const auto& entry : scripts) {
if (!HaveCScript(CScriptID(entry)) && !AddCScriptWithDB(batch, entry)) {
return false;
}
}
return true;
}
bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp)
{
WalletBatch batch(*database);
for (const auto& entry : privkey_map) {
const CKey& key = entry.second;
CPubKey pubkey = key.GetPubKey();
const CKeyID& id = entry.first;
assert(key.VerifyPubKey(pubkey));
mapKeyMetadata[id].nCreateTime = timestamp;
// If the private key is not present in the wallet, insert it.
if (!HaveKey(id) && !AddKeyPubKeyWithDB(batch, key, pubkey)) {
return false;
}
UpdateTimeFirstKey(timestamp);
}
return true;
}
bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp)
{
WalletBatch batch(*database);
for (const auto& entry : key_origins) {
AddKeyOriginWithDB(batch, entry.second.first, entry.second.second);
}
for (const CKeyID& id : ordered_pubkeys) {
auto entry = pubkey_map.find(id);
if (entry == pubkey_map.end()) {
continue;
}
const CPubKey& pubkey = entry->second;
CPubKey temp;
if (!GetPubKey(id, temp) && !AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) {
return false;
}
mapKeyMetadata[id].nCreateTime = timestamp;
// Add to keypool only works with pubkeys
if (add_keypool) {
AddKeypoolPubkeyWithDB(pubkey, internal, batch);
NotifyCanGetAddressesChanged();
}
}
return true;
}
bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool internal, const int64_t timestamp)
{
WalletBatch batch(*database);
for (const CScript& script : script_pub_keys) {
if (!have_solving_data || !::IsMine(*this, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated
if (!AddWatchOnlyWithDB(batch, script, timestamp)) {
return false;
}
}
CTxDestination dest;
ExtractDestination(script, dest);
if (!internal && IsValidDestination(dest)) {
SetAddressBookWithDB(batch, dest, label, "receive");
}
}
return true;
}
int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig)
{
std::vector<CTxOut> txouts;
@@ -3149,8 +3236,7 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx)
return DBErrors::LOAD_OK;
}
bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose)
bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose)
{
bool fUpdated = false;
{
@@ -3163,9 +3249,15 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s
}
NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO,
strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) );
if (!strPurpose.empty() && !WalletBatch(*database).WritePurpose(EncodeDestination(address), strPurpose))
if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose))
return false;
return WalletBatch(*database).WriteName(EncodeDestination(address), strName);
return batch.WriteName(EncodeDestination(address), strName);
}
bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& strPurpose)
{
WalletBatch batch(*database);
return SetAddressBookWithDB(batch, address, strName, strPurpose);
}
bool CWallet::DelAddressBook(const CTxDestination& address)
@@ -3315,13 +3407,6 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize)
return true;
}
void CWallet::AddKeypoolPubkey(const CPubKey& pubkey, const bool internal)
{
WalletBatch batch(*database);
AddKeypoolPubkeyWithDB(pubkey, internal, batch);
NotifyCanGetAddressesChanged();
}
void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch)
{
LOCK(cs_wallet);
@@ -4443,12 +4528,12 @@ bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const
return true;
}
bool CWallet::AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info)
bool CWallet::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info)
{
LOCK(cs_wallet);
std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint);
mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path;
mapKeyMetadata[pubkey.GetID()].has_key_origin = true;
mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path);
return WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true);
return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true);
}