Merge bitcoin/bitcoin#28710: Remove the legacy wallet and BDB dependency

de054df6dc contrib: Remove legacy wallet RPCs from bash completions (Ava Chow) 5dff04a1bb legacy spkm: Make IsMine() and CanProvide() private and migration only (Ava Chow) c0f3f3264f wallet: Remove unused db functions (Ava Chow) 83af1a3cca wallet: Delete LegacySPKM (Ava Chow) 8ede6dea0c wallet, rpc: Remove legacy wallet only RPCs (Ava Chow) 4de3cec28d test: rpcs disabled for descriptor wallets will be removed (Ava Chow) 84f671b01d test: Run multisig script limit test (Ava Chow) 810476f31e test: Remove unused options and variables, correct comments (Ava Chow) 04a7a7a28c build, wallet, doc: Remove BDB (Ava Chow) Pull request description: The final step of #20160. A bare minimum of legacy wallet code is kept in order to perform wallet migration. Migration of legacy wallets uses the independent BDB parser and a minimal `LegacyDataSPKM` that allows the legacy data to be loaded so that the migration can be completed. BDB has been removed as a dependency and documentation have been updated to reflect that. ACKs for top commit: Sjors: re-ACK de054df6dc maflcko: re-ACK de054df6dc 🔗 w0xlt: reACK de054df6dc rkrux: Concept ACK de054df6dc Tree-SHA512: 16a6c265bc1ada5e7a5ef9b95f0ff65015672ca46d9a43b7e10d60e9e085052e9bbfe01ac3e494cc606afb652a1b476b10e434d13e9877b67d2cb0196a9bd190
2026-06-14 16:50:17 +02:00 · 2025-05-07 15:19:17 +01:00
parent 6d5edfcc58 de054df6dc
commit efac285a0d
96 changed files with 110 additions and 6356 deletions
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -16,7 +16,6 @@ function(create_test_config)
  endmacro()

  set_configure_variable(ENABLE_WALLET ENABLE_WALLET)
-  set_configure_variable(WITH_BDB USE_BDB)
  set_configure_variable(BUILD_CLI BUILD_BITCOIN_CLI)
  set_configure_variable(BUILD_UTIL BUILD_BITCOIN_UTIL)
  set_configure_variable(BUILD_UTIL_CHAINSTATE BUILD_BITCOIN_CHAINSTATE)
--- a/test/config.ini.in
+++ b/test/config.ini.in
@@ -16,7 +16,6 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
 [components]
 # Which components are enabled. These are commented out by cmake if they were disabled during configuration.
@ENABLE_WALLET_TRUE@ENABLE_WALLET=true
-@USE_BDB_TRUE@USE_BDB=true
@BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true
@BUILD_BITCOIN_UTIL_TRUE@ENABLE_BITCOIN_UTIL=true
@BUILD_BITCOIN_CHAINSTATE_TRUE@ENABLE_BITCOIN_CHAINSTATE=true
--- a/test/functional/feature_bip68_sequence.py
+++ b/test/functional/feature_bip68_sequence.py
@@ -55,7 +55,6 @@ class BIP68Test(BitcoinTestFramework):
                '-testactivationheight=csv@432',
            ],
        ]
-        self.uses_wallet = None

    def run_test(self):
        self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
--- a/test/functional/rpc_createmultisig.py
+++ b/test/functional/rpc_createmultisig.py
@@ -51,6 +51,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework):
            for output_type in ["bech32", "p2sh-segwit", "legacy"]:
                self.do_multisig(keys, sigs, output_type)

+        self.test_multisig_script_limit()
        self.test_mixing_uncompressed_and_compressed_keys(node0)
        self.test_sortedmulti_descriptors_bip67()

--- a/test/functional/rpc_rawtransaction.py
+++ b/test/functional/rpc_rawtransaction.py
@@ -74,7 +74,6 @@ class RawTransactionsTest(BitcoinTestFramework):
        # whitelist peers to speed up tx relay / mempool sync
        self.noban_tx_relay = True
        self.supports_cli = False
-        self.uses_wallet = None

    def setup_network(self):
        super().setup_network()
--- a/test/functional/test_framework/bdb.py
+++ b/test/functional/test_framework/bdb.py
@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020-2021 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-"""
-Utilities for working directly with the wallet's BDB database file
-
-This is specific to the configuration of BDB used in this project:
-    - Outer database contains single subdatabase named 'main'
-    - btree
-    - btree internal, leaf and overflow pages
-
-Each key-value pair is two entries in a btree leaf, which optionally refers to overflow pages
-if the data doesn't fit into a single page. The first entry is the key, the one that follows
-is the value. And so on. Note that the entry data is itself not in the correct order. Instead
-entry offsets are stored in the correct order and those offsets are needed to then retrieve
-the data itself. Note that this implementation currently only supports reading databases that
-are in the same endianness as the host.
-
-Page format can be found in BDB source code dbinc/db_page.h
-
-`db_dump -da wallet.dat` is useful to see the data in a wallet.dat BDB file
-"""
-
-import struct
-
-# Important constants
-PAGE_HEADER_SIZE = 26
-OUTER_META_PAGE = 0
-
-# Page type values
-BTREE_INTERNAL = 3
-BTREE_LEAF = 5
-OVERFLOW_DATA = 7
-BTREE_META = 9
-
-# Record type values
-RECORD_KEYDATA = 1
-RECORD_OVERFLOW_DATA = 3
-
-# Some magic numbers for sanity checking
-BTREE_MAGIC = 0x053162
-DB_VERSION = 9
-SUBDATABASE_NAME = b'main'
-
-# Deserializes an internal, leaf or overflow page into a dict.
-# In addition to the common page header fields, the result contains an 'entries'
-# array of dicts with the following fields, depending on the page type:
-#     internal page [BTREE_INTERNAL]:
-#         - 'page_num': referenced page number (used to find further pages to process)
-#     leaf page [BTREE_LEAF]:
-#         - 'record_type': record type, must be RECORD_KEYDATA or RECORD_OVERFLOW_DATA
-#         - 'data': binary data (key or value payload),  if record type is RECORD_KEYDATA
-#         - 'page_num': referenced overflow page number, if record type is RECORD_OVERFLOW_DATA
-#     overflow page [OVERFLOW_DATA]:
-#         - 'data': binary data (part of key or value payload)
-def dump_page(data):
-    page_info = {}
-    page_header = data[0:26]
-    _, pgno, prev_pgno, next_pgno, entries, hf_offset, level, pg_type = struct.unpack('QIIIHHBB', page_header)
-    page_info['pgno'] = pgno
-    page_info['prev_pgno'] = prev_pgno
-    page_info['next_pgno'] = next_pgno
-    page_info['hf_offset'] = hf_offset
-    page_info['level'] = level
-    page_info['pg_type'] = pg_type
-    page_info['entry_offsets'] = struct.unpack('{}H'.format(entries), data[26:26 + entries * 2])
-    page_info['entries'] = []
-
-    assert pg_type in (BTREE_INTERNAL, BTREE_LEAF, OVERFLOW_DATA)
-
-    if pg_type == OVERFLOW_DATA:
-        assert entries == 1
-        page_info['entries'].append({'data': data[26:26 + hf_offset]})
-        return page_info
-
-    for i in range(0, entries):
-        entry = {}
-        offset = page_info['entry_offsets'][i]
-        record_header = data[offset:offset + 3]
-        offset += 3
-        e_len, record_type = struct.unpack('HB', record_header)
-
-        if pg_type == BTREE_INTERNAL:
-            assert record_type == RECORD_KEYDATA
-            internal_record_data = data[offset:offset + 9]
-            _, page_num, _ = struct.unpack('=BII', internal_record_data)
-            entry['page_num'] = page_num
-        elif pg_type == BTREE_LEAF:
-            assert record_type in (RECORD_KEYDATA, RECORD_OVERFLOW_DATA)
-            entry['record_type'] = record_type
-            if record_type == RECORD_KEYDATA:
-                entry['data'] = data[offset:offset + e_len]
-            elif record_type == RECORD_OVERFLOW_DATA:
-                overflow_record_data = data[offset:offset + 9]
-                _, page_num, _ = struct.unpack('=BII', overflow_record_data)
-                entry['page_num'] = page_num
-
-        page_info['entries'].append(entry)
-
-    return page_info
-
-# Deserializes a btree metadata page into a dict.
-# Does a simple sanity check on the magic value, type, and version
-def dump_meta_page(page):
-    # metadata page
-    # general metadata
-    metadata = {}
-    meta_page = page[0:72]
-    _, pgno, magic, version, pagesize, encrypt_alg, pg_type, metaflags, _, free, last_pgno, nparts, key_count, record_count, flags, uid = struct.unpack('QIIIIBBBBIIIIII20s', meta_page)
-    metadata['pgno'] = pgno
-    metadata['magic'] = magic
-    metadata['version'] = version
-    metadata['pagesize'] = pagesize
-    metadata['encrypt_alg'] = encrypt_alg
-    metadata['pg_type'] = pg_type
-    metadata['metaflags'] = metaflags
-    metadata['free'] = free
-    metadata['last_pgno'] = last_pgno
-    metadata['nparts'] = nparts
-    metadata['key_count'] = key_count
-    metadata['record_count'] = record_count
-    metadata['flags'] = flags
-    metadata['uid'] = uid.hex().encode()
-
-    assert magic == BTREE_MAGIC, 'bdb magic does not match bdb btree magic'
-    assert pg_type == BTREE_META, 'Metadata page is not a btree metadata page'
-    assert version == DB_VERSION, 'Database too new'
-
-    # btree metadata
-    btree_meta_page = page[72:512]
-    _, minkey, re_len, re_pad, root, _, crypto_magic, _, iv, chksum = struct.unpack('IIIII368sI12s16s20s', btree_meta_page)
-    metadata['minkey'] = minkey
-    metadata['re_len'] = re_len
-    metadata['re_pad'] = re_pad
-    metadata['root'] = root
-    metadata['crypto_magic'] = crypto_magic
-    metadata['iv'] = iv.hex().encode()
-    metadata['chksum'] = chksum.hex().encode()
-
-    return metadata
-
-# Given the dict from dump_leaf_page, get the key-value pairs and put them into a dict
-def extract_kv_pairs(page_data, pages):
-    out = {}
-    last_key = None
-    for i, entry in enumerate(page_data['entries']):
-        data = b''
-        if entry['record_type'] == RECORD_KEYDATA:
-            data = entry['data']
-        elif entry['record_type'] == RECORD_OVERFLOW_DATA:
-            next_page = entry['page_num']
-            while next_page != 0:
-                opage = pages[next_page]
-                opage_info = dump_page(opage)
-                data += opage_info['entries'][0]['data']
-                next_page = opage_info['next_pgno']
-
-        # By virtue of these all being pairs, even number entries are keys, and odd are values
-        if i % 2 == 0:
-            out[entry['data']] = b''
-            last_key = data
-        else:
-            out[last_key] = data
-    return out
-
-# Extract the key-value pairs of the BDB file given in filename
-def dump_bdb_kv(filename):
-    # Read in the BDB file and start deserializing it
-    pages = []
-    with open(filename, 'rb') as f:
-        # Determine pagesize first
-        data = f.read(PAGE_HEADER_SIZE)
-        pagesize = struct.unpack('I', data[20:24])[0]
-        assert pagesize in (512, 1024, 2048, 4096, 8192, 16384, 32768, 65536)
-
-        # Read rest of first page
-        data += f.read(pagesize - PAGE_HEADER_SIZE)
-        assert len(data) == pagesize
-
-        # Read all remaining pages
-        while len(data) > 0:
-            pages.append(data)
-            data = f.read(pagesize)
-
-    # Sanity check the meta pages, read root page
-    outer_meta_info = dump_meta_page(pages[OUTER_META_PAGE])
-    root_page_info = dump_page(pages[outer_meta_info['root']])
-    assert root_page_info['pg_type'] == BTREE_LEAF
-    assert len(root_page_info['entries']) == 2
-    assert root_page_info['entries'][0]['data'] == SUBDATABASE_NAME
-    assert len(root_page_info['entries'][1]['data']) == 4
-    inner_meta_page = int.from_bytes(root_page_info['entries'][1]['data'], 'big')
-    inner_meta_info = dump_meta_page(pages[inner_meta_page])
-
-    # Fetch the kv pairs from the pages
-    kv = {}
-    pages_to_process = [inner_meta_info['root']]
-    while len(pages_to_process) > 0:
-        curr_page_no = pages_to_process.pop()
-        assert curr_page_no <= outer_meta_info['last_pgno']
-        info = dump_page(pages[curr_page_no])
-        assert info['pg_type'] in (BTREE_INTERNAL, BTREE_LEAF)
-        if info['pg_type'] == BTREE_INTERNAL:
-            for entry in info['entries']:
-                pages_to_process.append(entry['page_num'])
-        elif info['pg_type'] == BTREE_LEAF:
-            info_kv = extract_kv_pairs(info, pages)
-            kv = {**kv, **info_kv}
-    return kv
--- a/test/functional/test_framework/test_framework.py
+++ b/test/functional/test_framework/test_framework.py
@@ -161,7 +161,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
        self.wallet_names = None
        # By default the wallet is not required. Set to true by skip_if_no_wallet().
        # Can also be set to None to indicate that the wallet will be used if available.
-        # When False or None, we ignore wallet_names regardless of what it is.
+        # When False or None, we ignore wallet_names in setup_nodes().
        self.uses_wallet = False
        # Disable ThreadOpenConnections by default, so that adding entries to
        # addrman will not result in automatic connections to them.
@@ -487,10 +487,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
                n.createwallet(wallet_name=wallet_name, load_on_startup=True)
            n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase', rescan=True)

-    # Only enables wallet support when the module is available
-    def enable_wallet_if_possible(self):
-        self._requires_wallet = self.is_wallet_compiled()
-
    def run_test(self):
        """Tests must override this method to define test logic"""
        raise NotImplementedError
@@ -982,11 +978,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
        if not self.is_wallet_compiled():
            raise SkipTest("wallet has not been compiled.")

-    def skip_if_no_bdb(self):
-        """Skip the running test if BDB has not been compiled."""
-        if not self.is_bdb_compiled():
-            raise SkipTest("BDB has not been compiled.")
-
    def skip_if_no_wallet_tool(self):
        """Skip the running test if bitcoin-wallet has not been compiled."""
        if not self.is_wallet_tool_compiled():
@@ -1057,9 +1048,5 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
        """Checks whether the USDT tracepoints were compiled."""
        return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS")

-    def is_bdb_compiled(self):
-        """Checks whether the wallet module was compiled with BDB support."""
-        return self.config["components"].getboolean("USE_BDB")
-
    def has_blockfile(self, node, filenum: str):
        return (node.blocks_path/ f"blk{filenum}.dat").is_file()
--- a/test/functional/test_framework/util.py
+++ b/test/functional/test_framework/util.py
@@ -459,7 +459,6 @@ def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect=
        f.write("printtoconsole=0\n")
        f.write("natpmp=0\n")
        f.write("shrinkdebugfile=0\n")
-        f.write("deprecatedrpc=create_bdb\n")  # Required to run the tests
        # To improve SQLite wallet performance so that the tests don't timeout, use -unsafesqlitesync
        f.write("unsafesqlitesync=1\n")
        if disable_autoconnect:
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -29,8 +29,6 @@ import tempfile
 import re
 import logging

-os.environ["REQUIRE_WALLET_TYPE_SET"] = "1"
-
 # Minimum amount of space to run the tests.
 MIN_FREE_SPACE = 1.1 * 1024 * 1024 * 1024
 # Additional space to run an extra job.
--- a/test/functional/tool_wallet.py
+++ b/test/functional/tool_wallet.py
@@ -20,16 +20,10 @@ from test_framework.util import (


 class ToolWalletTest(BitcoinTestFramework):
-    def add_options(self, parser):
-        parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file")
-        parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally")
-
    def set_test_params(self):
        self.num_nodes = 1
        self.setup_clean_chain = True
        self.rpc_timeout = 120
-        if self.options.swap_bdb_endian:
-            self.extra_args = [["-swapbdbendian"]]

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()
@@ -37,8 +31,6 @@ class ToolWalletTest(BitcoinTestFramework):

    def bitcoin_wallet_process(self, *args):
        default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain]
-        if "dump" in args and self.options.bdbro:
-            default_args.append("-withinternalbdb")

        return subprocess.Popen(self.get_binaries().wallet_argv() + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)

--- a/test/functional/wallet_createwallet.py
+++ b/test/functional/wallet_createwallet.py
@@ -15,7 +15,6 @@ from test_framework.wallet_util import generate_keypair, WalletUnlock


 EMPTY_PASSPHRASE_MSG = "Empty string given as passphrase, wallet will not be encrypted."
-LEGACY_WALLET_MSG = "Wallet created successfully. The legacy wallet type is being deprecated and support for creating and opening legacy wallets will be removed in the future."


 class CreateWalletTest(BitcoinTestFramework):
@@ -27,7 +26,6 @@ class CreateWalletTest(BitcoinTestFramework):

    def run_test(self):
        node = self.nodes[0]
-        self.generate(node, 1) # Leave IBD for sethdseed

        self.log.info("Run createwallet with invalid parameters.")
        # Run createwallet with invalid parameters. This must not prevent a new wallet with the same name from being created with the correct parameters.
--- a/test/functional/wallet_descriptor.py
+++ b/test/functional/wallet_descriptor.py
@@ -137,18 +137,6 @@ class WalletDescriptorTest(BitcoinTestFramework):
        addr = recv_wrpc.getnewaddress()
        send_wrpc.sendtoaddress(addr, 10)

-        # Make sure things are disabled
-        self.log.info("Test disabled RPCs")
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importprivkey, "cVpF924EspNh8KjYsfhgY96mmxvT6DgdWiTYMtMjuM74hJaU5psW")
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importpubkey, send_wrpc.getaddressinfo(send_wrpc.getnewaddress())["pubkey"])
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importaddress, recv_wrpc.getnewaddress())
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importmulti, [])
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.addmultisigaddress, 1, [recv_wrpc.getnewaddress()])
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.dumpprivkey, recv_wrpc.getnewaddress())
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.dumpwallet, 'wallet.dump')
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.importwallet, 'wallet.dump')
-        assert_raises_rpc_error(-4, "Only legacy wallets are supported by this command", recv_wrpc.rpc.sethdseed)
-
        self.log.info("Test encryption")
        # Get the master fingerprint before encrypt
        info1 = send_wrpc.getaddressinfo(send_wrpc.getnewaddress())
--- a/test/functional/wallet_hd.py
+++ b/test/functional/wallet_hd.py
@@ -43,7 +43,6 @@ class WalletHDTest(BitcoinTestFramework):

        # This should be enough to keep the master key and the non-HD key
        self.nodes[1].backupwallet(self.nodes[1].datadir_path / "hd.bak")
-        #self.nodes[1].dumpwallet(self.nodes[1].datadir_path / "hd.dump")

        # Derive some HD addresses and remember the last
        # Also send funds to each add
--- a/test/functional/wallet_migration.py
+++ b/test/functional/wallet_migration.py
@@ -15,7 +15,6 @@ from test_framework.address import (
    script_to_p2sh,
    script_to_p2wsh,
 )
-from test_framework.bdb import BTREE_MAGIC
 from test_framework.descriptors import descsum_create
 from test_framework.key import ECPubKey
 from test_framework.test_framework import BitcoinTestFramework
@@ -33,6 +32,8 @@ from test_framework.wallet_util import (
    generate_keypair,
 )

+BTREE_MAGIC = 0x053162
+

 class WalletMigrationTest(BitcoinTestFramework):
    def set_test_params(self):
@@ -46,10 +47,14 @@ class WalletMigrationTest(BitcoinTestFramework):
        self.skip_if_no_previous_releases()

    def setup_nodes(self):
-        self.add_nodes(self.num_nodes, versions=[
-            None,
-            280000,
-        ])
+        self.add_nodes(
+            self.num_nodes,
+            extra_args=self.extra_args,
+            versions=[
+                None,
+                280000,
+            ],
+        )
        self.start_nodes()
        self.init_wallet(node=0)

--- a/test/lint/lint-locale-dependence.py
+++ b/test/lint/lint-locale-dependence.py
@@ -45,7 +45,6 @@ KNOWN_VIOLATIONS = [
    "src/dbwrapper.cpp:.*vsnprintf",
    "src/test/fuzz/locale.cpp:.*setlocale",
    "src/test/util_tests.cpp:.*strtoll",
-    "src/wallet/bdb.cpp:.*DbEnv::strerror",  # False positive
    "src/util/syserror.cpp:.*strerror",      # Outside this function use `SysErrorString`
 ]

--- a/test/sanitizer_suppressions/tsan
+++ b/test/sanitizer_suppressions/tsan
@@ -5,9 +5,6 @@

 # race (TODO fix)
 race:LoadWallet
-race:WalletBatch::WriteHDChain
-race:BerkeleyBatch
-race:BerkeleyDatabase
 race:DatabaseBatch
 race:zmq::*
 race:bitcoin-qt
@@ -25,8 +22,6 @@ race:src/qt/test/*
 deadlock:src/qt/test/*

 # External libraries
-# https://github.com/bitcoin/bitcoin/pull/27658#issuecomment-1547639621
-deadlock:libdb
 race:libzmq

 # Intermittent issues