Merge bitcoin/bitcoin#19602: wallet: Migrate legacy wallets to descriptor wallets

53e7ed075c doc: Release notes and other docs for migration (Andrew Chow)
9c44bfe244 Test migratewallet (Andrew Chow)
0b26e7cdf2 descriptors: addr() and raw() should return false for ToPrivateString (Andrew Chow)
31764c3f87 Add migratewallet RPC (Andrew Chow)
0bf7b38bff Implement MigrateLegacyToDescriptor (Andrew Chow)
e7b16f925a Implement MigrateToSQLite (Andrew Chow)
5b62f095e7 wallet: Refactor SetupDescSPKMs to take CExtKey (Andrew Chow)
22401f17e0 Implement LegacyScriptPubKeyMan::DeleteRecords (Andrew Chow)
35f428fae6 Implement LegacyScriptPubKeyMan::MigrateToDescriptor (Andrew Chow)
ea1ab390e4 scriptpubkeyman: Implement GetScriptPubKeys in Legacy (Andrew Chow)
e664af2976 Apply label to all scriptPubKeys of imported combo() (Andrew Chow)

Pull request description:

  This PR adds a new `migratewallet` RPC which migrates a legacy wallet to a descriptor wallet. Migrated wallets will need a new backup. If a wallet has watchonly stuff in it, a new watchonly descriptor wallet will be created containing those watchonly things. The related transactions, labels, and descriptors for those watchonly things will be removed from the original wallet. Migrated wallets will not have any of the legacy things be available for fetching from `getnewaddress` or `getrawchangeaddress`. Wallets that have private keys enabled will have newly generated descriptors. Wallets with private keys disabled will not have any active `ScriptPubKeyMan`s.

  For the basic HD wallet case of just generated keys, in addition to the standard descriptor wallet descriptors using the master key derived from the pre-existing hd seed, the migration will also create 3 descriptors for each HD chain in: a ranged combo external, a ranged combo internal, and a single key combo for the seed (the seed is a valid key that we can receive coins at!). The migrated wallet will then have newly generated descriptors as the active `ScriptPubKeyMan`s. This is equivalent to creating a new descriptor wallet and importing the 3 descriptors for each HD chain. For wallets containing non-HD keys, each key will have its own combo descriptor.

  There are also tests.

ACKs for top commit:
  Sjors:
    tACK 53e7ed075c
  w0xlt:
    reACK 53e7ed075c

Tree-SHA512: c0c003694ca2e17064922d08e8464278d314e970efb7df874b4fe04ec5d124c7206409ca701c65c099d17779ab2136ae63f1da2a9dba39b45f6d62cf93b5c60a
This commit is contained in:
Andrew Chow
2022-09-01 15:33:34 -04:00
15 changed files with 1458 additions and 34 deletions

View File

@@ -3434,6 +3434,29 @@ void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc)
}
}
void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key)
{
AssertLockHeld(cs_wallet);
for (bool internal : {false, true}) {
for (OutputType t : OUTPUT_TYPES) {
auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this));
if (IsCrypted()) {
if (IsLocked()) {
throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors");
}
if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) {
throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors");
}
}
spk_manager->SetupDescriptorGeneration(master_key, t, internal);
uint256 id = spk_manager->GetID();
m_spk_managers[id] = std::move(spk_manager);
AddActiveScriptPubKeyMan(id, t, internal);
}
}
}
void CWallet::SetupDescriptorScriptPubKeyMans()
{
AssertLockHeld(cs_wallet);
@@ -3449,23 +3472,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans()
CExtKey master_key;
master_key.SetSeed(seed_key);
for (bool internal : {false, true}) {
for (OutputType t : OUTPUT_TYPES) {
auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this));
if (IsCrypted()) {
if (IsLocked()) {
throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors");
}
if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) {
throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors");
}
}
spk_manager->SetupDescriptorGeneration(master_key, t, internal);
uint256 id = spk_manager->GetID();
m_spk_managers[id] = std::move(spk_manager);
AddActiveScriptPubKeyMan(id, t, internal);
}
}
SetupDescriptorScriptPubKeyMans(master_key);
} else {
ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner();
@@ -3633,9 +3640,13 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat
return nullptr;
}
CTxDestination dest;
if (!internal && ExtractDestination(script_pub_keys.at(0), dest)) {
SetAddressBook(dest, label, "receive");
if (!internal) {
for (const auto& script : script_pub_keys) {
CTxDestination dest;
if (ExtractDestination(script, dest)) {
SetAddressBook(dest, label, "receive");
}
}
}
}
@@ -3644,4 +3655,472 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat
return spk_man;
}
bool CWallet::MigrateToSQLite(bilingual_str& error)
{
AssertLockHeld(cs_wallet);
WalletLogPrintf("Migrating wallet storage database from BerkeleyDB to SQLite.\n");
if (m_database->Format() == "sqlite") {
error = _("Error: This wallet already uses SQLite");
return false;
}
// Get all of the records for DB type migration
std::unique_ptr<DatabaseBatch> batch = m_database->MakeBatch();
std::vector<std::pair<SerializeData, SerializeData>> records;
if (!batch->StartCursor()) {
error = _("Error: Unable to begin reading all records in the database");
return false;
}
bool complete = false;
while (true) {
CDataStream ss_key(SER_DISK, CLIENT_VERSION);
CDataStream ss_value(SER_DISK, CLIENT_VERSION);
bool ret = batch->ReadAtCursor(ss_key, ss_value, complete);
if (!ret) {
break;
}
SerializeData key(ss_key.begin(), ss_key.end());
SerializeData value(ss_value.begin(), ss_value.end());
records.emplace_back(key, value);
}
batch->CloseCursor();
batch.reset();
if (!complete) {
error = _("Error: Unable to read all records in the database");
return false;
}
// Close this database and delete the file
fs::path db_path = fs::PathFromString(m_database->Filename());
fs::path db_dir = db_path.parent_path();
m_database->Close();
fs::remove(db_path);
// Make new DB
DatabaseOptions opts;
opts.require_create = true;
opts.require_format = DatabaseFormat::SQLITE;
DatabaseStatus db_status;
std::unique_ptr<WalletDatabase> new_db = MakeDatabase(db_dir, opts, db_status, error);
assert(new_db); // This is to prevent doing anything further with this wallet. The original file was deleted, but a backup exists.
m_database.reset();
m_database = std::move(new_db);
// Write existing records into the new DB
batch = m_database->MakeBatch();
bool began = batch->TxnBegin();
assert(began); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution.
for (const auto& [key, value] : records) {
CDataStream ss_key(key, SER_DISK, CLIENT_VERSION);
CDataStream ss_value(value, SER_DISK, CLIENT_VERSION);
if (!batch->Write(ss_key, ss_value)) {
batch->TxnAbort();
m_database->Close();
fs::remove(m_database->Filename());
assert(false); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution.
}
}
bool committed = batch->TxnCommit();
assert(committed); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution.
return true;
}
std::optional<MigrationData> CWallet::GetDescriptorsForLegacy(bilingual_str& error) const
{
AssertLockHeld(cs_wallet);
LegacyScriptPubKeyMan* legacy_spkm = GetLegacyScriptPubKeyMan();
if (!legacy_spkm) {
error = _("Error: This wallet is already a descriptor wallet");
return std::nullopt;
}
std::optional<MigrationData> res = legacy_spkm->MigrateToDescriptor();
if (res == std::nullopt) {
error = _("Error: Unable to produce descriptors for this legacy wallet. Make sure the wallet is unlocked first");
return std::nullopt;
}
return res;
}
bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error)
{
AssertLockHeld(cs_wallet);
LegacyScriptPubKeyMan* legacy_spkm = GetLegacyScriptPubKeyMan();
if (!legacy_spkm) {
error = _("Error: This wallet is already a descriptor wallet");
return false;
}
for (auto& desc_spkm : data.desc_spkms) {
if (m_spk_managers.count(desc_spkm->GetID()) > 0) {
error = _("Error: Duplicate descriptors created during migration. Your wallet may be corrupted.");
return false;
}
m_spk_managers[desc_spkm->GetID()] = std::move(desc_spkm);
}
// Remove the LegacyScriptPubKeyMan from disk
if (!legacy_spkm->DeleteRecords()) {
return false;
}
// Remove the LegacyScriptPubKeyMan from memory
m_spk_managers.erase(legacy_spkm->GetID());
m_external_spk_managers.clear();
m_internal_spk_managers.clear();
// Setup new descriptors
SetWalletFlag(WALLET_FLAG_DESCRIPTORS);
if (!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) {
// Use the existing master key if we have it
if (data.master_key.key.IsValid()) {
SetupDescriptorScriptPubKeyMans(data.master_key);
} else {
// Setup with a new seed if we don't.
SetupDescriptorScriptPubKeyMans();
}
}
// Check if the transactions in the wallet are still ours. Either they belong here, or they belong in the watchonly wallet.
// We need to go through these in the tx insertion order so that lookups to spends works.
std::vector<uint256> txids_to_delete;
for (const auto& [_pos, wtx] : wtxOrdered) {
if (!IsMine(*wtx->tx) && !IsFromMe(*wtx->tx)) {
// Check it is the watchonly wallet's
// solvable_wallet doesn't need to be checked because transactions for those scripts weren't being watched for
if (data.watchonly_wallet) {
LOCK(data.watchonly_wallet->cs_wallet);
if (data.watchonly_wallet->IsMine(*wtx->tx) || data.watchonly_wallet->IsFromMe(*wtx->tx)) {
// Add to watchonly wallet
if (!data.watchonly_wallet->AddToWallet(wtx->tx, wtx->m_state)) {
error = _("Error: Could not add watchonly tx to watchonly wallet");
return false;
}
// Mark as to remove from this wallet
txids_to_delete.push_back(wtx->GetHash());
continue;
}
}
// Both not ours and not in the watchonly wallet
error = strprintf(_("Error: Transaction %s in wallet cannot be identified to belong to migrated wallets"), wtx->GetHash().GetHex());
return false;
}
}
// Do the removes
if (txids_to_delete.size() > 0) {
std::vector<uint256> deleted_txids;
if (ZapSelectTx(txids_to_delete, deleted_txids) != DBErrors::LOAD_OK) {
error = _("Error: Could not delete watchonly transactions");
return false;
}
if (deleted_txids != txids_to_delete) {
error = _("Error: Not all watchonly txs could be deleted");
return false;
}
// Tell the GUI of each tx
for (const uint256& txid : deleted_txids) {
NotifyTransactionChanged(txid, CT_UPDATED);
}
}
// Check the address book data in the same way we did for transactions
std::vector<CTxDestination> dests_to_delete;
for (const auto& addr_pair : m_address_book) {
// Labels applied to receiving addresses should go based on IsMine
if (addr_pair.second.purpose == "receive") {
if (!IsMine(addr_pair.first)) {
// Check the address book data is the watchonly wallet's
if (data.watchonly_wallet) {
LOCK(data.watchonly_wallet->cs_wallet);
if (data.watchonly_wallet->IsMine(addr_pair.first)) {
// Add to the watchonly. Preserve the labels, purpose, and change-ness
std::string label = addr_pair.second.GetLabel();
std::string purpose = addr_pair.second.purpose;
if (!purpose.empty()) {
data.watchonly_wallet->m_address_book[addr_pair.first].purpose = purpose;
}
if (!addr_pair.second.IsChange()) {
data.watchonly_wallet->m_address_book[addr_pair.first].SetLabel(label);
}
dests_to_delete.push_back(addr_pair.first);
continue;
}
}
if (data.solvable_wallet) {
LOCK(data.solvable_wallet->cs_wallet);
if (data.solvable_wallet->IsMine(addr_pair.first)) {
// Add to the solvable. Preserve the labels, purpose, and change-ness
std::string label = addr_pair.second.GetLabel();
std::string purpose = addr_pair.second.purpose;
if (!purpose.empty()) {
data.solvable_wallet->m_address_book[addr_pair.first].purpose = purpose;
}
if (!addr_pair.second.IsChange()) {
data.solvable_wallet->m_address_book[addr_pair.first].SetLabel(label);
}
dests_to_delete.push_back(addr_pair.first);
continue;
}
}
// Not ours, not in watchonly wallet, and not in solvable
error = _("Error: Address book data in wallet cannot be identified to belong to migrated wallets");
return false;
}
} else {
// Labels for everything else (send) should be cloned to all
if (data.watchonly_wallet) {
LOCK(data.watchonly_wallet->cs_wallet);
// Add to the watchonly. Preserve the labels, purpose, and change-ness
std::string label = addr_pair.second.GetLabel();
std::string purpose = addr_pair.second.purpose;
if (!purpose.empty()) {
data.watchonly_wallet->m_address_book[addr_pair.first].purpose = purpose;
}
if (!addr_pair.second.IsChange()) {
data.watchonly_wallet->m_address_book[addr_pair.first].SetLabel(label);
}
continue;
}
if (data.solvable_wallet) {
LOCK(data.solvable_wallet->cs_wallet);
// Add to the solvable. Preserve the labels, purpose, and change-ness
std::string label = addr_pair.second.GetLabel();
std::string purpose = addr_pair.second.purpose;
if (!purpose.empty()) {
data.solvable_wallet->m_address_book[addr_pair.first].purpose = purpose;
}
if (!addr_pair.second.IsChange()) {
data.solvable_wallet->m_address_book[addr_pair.first].SetLabel(label);
}
continue;
}
}
}
// Remove the things to delete
if (dests_to_delete.size() > 0) {
for (const auto& dest : dests_to_delete) {
if (!DelAddressBook(dest)) {
error = _("Error: Unable to remove watchonly address book data");
return false;
}
}
}
// Connect the SPKM signals
ConnectScriptPubKeyManNotifiers();
NotifyCanGetAddressesChanged();
WalletLogPrintf("Wallet migration complete.\n");
return true;
}
bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error, MigrationResult& res) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet)
{
AssertLockHeld(wallet.cs_wallet);
// Get all of the descriptors from the legacy wallet
std::optional<MigrationData> data = wallet.GetDescriptorsForLegacy(error);
if (data == std::nullopt) return false;
// Create the watchonly and solvable wallets if necessary
if (data->watch_descs.size() > 0 || data->solvable_descs.size() > 0) {
DatabaseOptions options;
options.require_existing = false;
options.require_create = true;
// Make the wallets
options.create_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_DESCRIPTORS;
if (wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)) {
options.create_flags |= WALLET_FLAG_AVOID_REUSE;
}
if (wallet.IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) {
options.create_flags |= WALLET_FLAG_KEY_ORIGIN_METADATA;
}
if (data->watch_descs.size() > 0) {
wallet.WalletLogPrintf("Making a new watchonly wallet containing the watched scripts\n");
DatabaseStatus status;
std::vector<bilingual_str> warnings;
std::string wallet_name = wallet.GetName() + "_watchonly";
data->watchonly_wallet = CreateWallet(context, wallet_name, std::nullopt, options, status, error, warnings);
if (status != DatabaseStatus::SUCCESS) {
error = _("Error: Failed to create new watchonly wallet");
return false;
}
res.watchonly_wallet = data->watchonly_wallet;
LOCK(data->watchonly_wallet->cs_wallet);
// Parse the descriptors and add them to the new wallet
for (const auto& [desc_str, creation_time] : data->watch_descs) {
// Parse the descriptor
FlatSigningProvider keys;
std::string parse_err;
std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, parse_err, /* require_checksum */ true);
assert(desc); // It shouldn't be possible to have the LegacyScriptPubKeyMan make an invalid descriptor
assert(!desc->IsRange()); // It shouldn't be possible to have LegacyScriptPubKeyMan make a ranged watchonly descriptor
// Add to the wallet
WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0);
data->watchonly_wallet->AddWalletDescriptor(w_desc, keys, "", false);
}
// Add the wallet to settings
UpdateWalletSetting(*context.chain, wallet_name, /*load_on_startup=*/true, warnings);
}
if (data->solvable_descs.size() > 0) {
wallet.WalletLogPrintf("Making a new watchonly wallet containing the unwatched solvable scripts\n");
DatabaseStatus status;
std::vector<bilingual_str> warnings;
std::string wallet_name = wallet.GetName() + "_solvables";
data->solvable_wallet = CreateWallet(context, wallet_name, std::nullopt, options, status, error, warnings);
if (status != DatabaseStatus::SUCCESS) {
error = _("Error: Failed to create new watchonly wallet");
return false;
}
res.solvables_wallet = data->solvable_wallet;
LOCK(data->solvable_wallet->cs_wallet);
// Parse the descriptors and add them to the new wallet
for (const auto& [desc_str, creation_time] : data->solvable_descs) {
// Parse the descriptor
FlatSigningProvider keys;
std::string parse_err;
std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, parse_err, /* require_checksum */ true);
assert(desc); // It shouldn't be possible to have the LegacyScriptPubKeyMan make an invalid descriptor
assert(!desc->IsRange()); // It shouldn't be possible to have LegacyScriptPubKeyMan make a ranged watchonly descriptor
// Add to the wallet
WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0);
data->solvable_wallet->AddWalletDescriptor(w_desc, keys, "", false);
}
// Add the wallet to settings
UpdateWalletSetting(*context.chain, wallet_name, /*load_on_startup=*/true, warnings);
}
}
// Add the descriptors to wallet, remove LegacyScriptPubKeyMan, and cleanup txs and address book data
if (!wallet.ApplyMigrationData(*data, error)) {
return false;
}
return true;
}
util::Result<MigrationResult> MigrateLegacyToDescriptor(std::shared_ptr<CWallet>&& wallet, WalletContext& context)
{
MigrationResult res;
bilingual_str error;
std::vector<bilingual_str> warnings;
// Make a backup of the DB
std::string wallet_name = wallet->GetName();
fs::path this_wallet_dir = fs::absolute(fs::PathFromString(wallet->GetDatabase().Filename())).parent_path();
fs::path backup_filename = fs::PathFromString(strprintf("%s-%d.legacy.bak", wallet_name, GetTime()));
fs::path backup_path = this_wallet_dir / backup_filename;
if (!wallet->BackupWallet(fs::PathToString(backup_path))) {
return util::Error{_("Error: Unable to make a backup of your wallet")};
}
res.backup_path = backup_path;
// Unload the wallet so that nothing else tries to use it while we're changing it
if (!RemoveWallet(context, wallet, /*load_on_start=*/std::nullopt, warnings)) {
return util::Error{_("Unable to unload the wallet before migrating")};
}
UnloadWallet(std::move(wallet));
// Load the wallet but only in the context of this function.
// No signals should be connected nor should anything else be aware of this wallet
WalletContext empty_context;
empty_context.args = context.args;
DatabaseOptions options;
options.require_existing = true;
DatabaseStatus status;
std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(wallet_name, options, status, error);
if (!database) {
return util::Error{Untranslated("Wallet file verification failed.") + Untranslated(" ") + error};
}
std::shared_ptr<CWallet> local_wallet = CWallet::Create(empty_context, wallet_name, std::move(database), options.create_flags, error, warnings);
if (!local_wallet) {
return util::Error{Untranslated("Wallet loading failed.") + Untranslated(" ") + error};
}
bool success = false;
{
LOCK(local_wallet->cs_wallet);
// First change to using SQLite
if (!local_wallet->MigrateToSQLite(error)) return util::Error{error};
// Do the migration, and cleanup if it fails
success = DoMigration(*local_wallet, context, error, res);
}
if (success) {
// Migration successful, unload the wallet locally, then reload it.
assert(local_wallet.use_count() == 1);
local_wallet.reset();
LoadWallet(context, wallet_name, /*load_on_start=*/std::nullopt, options, status, error, warnings);
res.wallet_name = wallet_name;
} else {
// Migration failed, cleanup
// Copy the backup to the actual wallet dir
fs::path temp_backup_location = fsbridge::AbsPathJoin(GetWalletDir(), backup_filename);
fs::copy_file(backup_path, temp_backup_location, fs::copy_options::none);
// Remember this wallet's walletdir to remove after unloading
std::vector<fs::path> wallet_dirs;
wallet_dirs.push_back(fs::PathFromString(local_wallet->GetDatabase().Filename()).parent_path());
// Unload the wallet locally
assert(local_wallet.use_count() == 1);
local_wallet.reset();
// Make list of wallets to cleanup
std::vector<std::shared_ptr<CWallet>> created_wallets;
created_wallets.push_back(std::move(res.watchonly_wallet));
created_wallets.push_back(std::move(res.solvables_wallet));
// Get the directories to remove after unloading
for (std::shared_ptr<CWallet>& w : created_wallets) {
wallet_dirs.push_back(fs::PathFromString(w->GetDatabase().Filename()).parent_path());
}
// Unload the wallets
for (std::shared_ptr<CWallet>& w : created_wallets) {
if (!RemoveWallet(context, w, /*load_on_start=*/false)) {
error += _("\nUnable to cleanup failed migration");
return util::Error{error};
}
UnloadWallet(std::move(w));
}
// Delete the wallet directories
for (fs::path& dir : wallet_dirs) {
fs::remove_all(dir);
}
// Restore the backup
DatabaseStatus status;
std::vector<bilingual_str> warnings;
if (!RestoreWallet(context, temp_backup_location, wallet_name, /*load_on_start=*/std::nullopt, status, error, warnings)) {
error += _("\nUnable to restore backup of wallet.");
return util::Error{error};
}
// Move the backup to the wallet dir
fs::copy_file(temp_backup_location, backup_path, fs::copy_options::none);
fs::remove(temp_backup_location);
return util::Error{error};
}
return res;
}
} // namespace wallet