Merge bitcoin/bitcoin#26606: wallet: Implement independent BDB parser

d51fbab4b3 wallet, test: Be able to always swap BDB endianness (Ava Chow)
0b753156ce test: Test bdb_ro dump of wallet without reset LSNs (Ava Chow)
c1984f1282 test: Test dumping dbs with overflow pages (Ava Chow)
fd7b16e391 test: Test dumps of other endian BDB files (Ava Chow)
6ace3e953f bdb: Be able to make byteswapped databases (Ava Chow)
d9878903fb Error if LSNs are not reset (Ava Chow)
4d7a3ae78e Berkeley RO Database fuzz test (TheCharlatan)
3568dce9e9 tests: Add BerkeleyRO to db prefix tests (Ava Chow)
70cfbfdadf wallettool: Optionally use BERKELEY_RO as format when dumping BDB wallets (Ava Chow)
dd57713f6e Add MakeBerkeleyRODatabase (Ava Chow)
6e50bee67d Implement handling of other endianness in BerkeleyRODatabase (Ava Chow)
cdd61c9cc1 wallet: implement independent BDB deserializer in BerkeleyRODatabase (Ava Chow)
ecba230979 wallet: implement BerkeleyRODatabase::Backup (Ava Chow)
0c8e728476 wallet: implement BerkeleyROBatch (Ava Chow)
756ff9b478 wallet: add dummy BerkeleyRODatabase and BerkeleyROBatch classes (Ava Chow)
ca18aea5c4 Add AutoFile::seek and tell (Ava Chow)

Pull request description:

  Split from #26596

  This PR adds `BerkeleyRODatabase` which is an independent implementation of a BDB file parser. It provides read only access to a BDB file, and can therefore be used as a read only database backend for wallets. This will be used for dumping legacy wallet records and migrating legacy wallets without the need for BDB itself.

  Wallettool's `dump` command is changed to use `BerkeleyRODatabase` instead of `BerkeleyDatabase` (and `CWallet` itself) to demonstrate that this parser works and to test it against the existing wallettool functional tests.

ACKs for top commit:
  josibake:
    reACK d51fbab4b3
  TheCharlatan:
    Re-ACK d51fbab4b3
  furszy:
    reACK d51fbab4b3
  laanwj:
    re-ACK d51fbab4b3
  theStack:
    ACK d51fbab4b3

Tree-SHA512: 1e7b97edf223b2974eed2e9eac1179fc82bb6359e0a66b7d2a0c8b9fa515eae9ea036f1edf7c76cdab2e75ad994962b134b41056ccfbc33b8d54f0859e86657b

test/functional/test_framework/test_node.py

@@ -419,8 +419,9 @@ class TestNode():
         return True
 
     def wait_until_stopped(self, *, timeout=BITCOIND_PROC_WAIT_TIMEOUT, expect_error=False, **kwargs):
-        expected_ret_code = 1 if expect_error else 0  # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS
-        self.wait_until(lambda: self.is_node_stopped(expected_ret_code=expected_ret_code, **kwargs), timeout=timeout)
+        if "expected_ret_code" not in kwargs:
+            kwargs["expected_ret_code"] = 1 if expect_error else 0  # Whether node shutdown return EXIT_FAILURE or EXIT_SUCCESS
+        self.wait_until(lambda: self.is_node_stopped(**kwargs), timeout=timeout)
 
     def replace_in_config(self, replacements):
         """

test/functional/test_runner.py

@@ -183,6 +183,8 @@ BASE_SCRIPTS = [
     'mempool_resurrect.py',
     'wallet_txn_doublespend.py --mineblock',
     'tool_wallet.py --legacy-wallet',
+    'tool_wallet.py --legacy-wallet --bdbro',
+    'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian',
     'tool_wallet.py --descriptors',
     'tool_signet_miner.py --legacy-wallet',
     'tool_signet_miner.py --descriptors',

test/functional/tool_wallet.py

@@ -5,6 +5,7 @@
 """Test bitcoin-wallet."""
 
 import os
+import platform
 import stat
 import subprocess
 import textwrap
@@ -14,6 +15,7 @@ from collections import OrderedDict
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
+    assert_greater_than,
     sha256sum_file,
 )
 
@@ -21,11 +23,15 @@ from test_framework.util import (
 class ToolWalletTest(BitcoinTestFramework):
     def add_options(self, parser):
         self.add_wallet_options(parser)
+        parser.add_argument("--bdbro", action="store_true", help="Use the BerkeleyRO internal parser when dumping a Berkeley DB wallet file")
+        parser.add_argument("--swap-bdb-endian", action="store_true",help="When making Legacy BDB wallets, always make then byte swapped internally")
 
     def set_test_params(self):
         self.num_nodes = 1
         self.setup_clean_chain = True
         self.rpc_timeout = 120
+        if self.options.swap_bdb_endian:
+            self.extra_args = [["-swapbdbendian"]]
 
     def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
@@ -35,15 +41,21 @@ class ToolWalletTest(BitcoinTestFramework):
         default_args = ['-datadir={}'.format(self.nodes[0].datadir_path), '-chain=%s' % self.chain]
         if not self.options.descriptors and 'create' in args:
             default_args.append('-legacy')
+        if "dump" in args and self.options.bdbro:
+            default_args.append("-withinternalbdb")
 
         return subprocess.Popen([self.options.bitcoinwallet] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
 
     def assert_raises_tool_error(self, error, *args):
         p = self.bitcoin_wallet_process(*args)
         stdout, stderr = p.communicate()
-        assert_equal(p.poll(), 1)
         assert_equal(stdout, '')
-        assert_equal(stderr.strip(), error)
+        if isinstance(error, tuple):
+            assert_equal(p.poll(), error[0])
+            assert error[1] in stderr.strip()
+        else:
+            assert_equal(p.poll(), 1)
+            assert error in stderr.strip()
 
     def assert_tool_output(self, output, *args):
         p = self.bitcoin_wallet_process(*args)
@@ -451,6 +463,88 @@ class ToolWalletTest(BitcoinTestFramework):
         ''')
         self.assert_tool_output(expected_output, "-wallet=conflicts", "info")
 
+    def test_dump_endianness(self):
+        self.log.info("Testing dumps of the same contents with different BDB endianness")
+
+        self.start_node(0)
+        self.nodes[0].createwallet("endian")
+        self.stop_node(0)
+
+        wallet_dump = self.nodes[0].datadir_path / "endian.dump"
+        self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=endian", f"-dumpfile={wallet_dump}", "dump")
+        expected_dump = self.read_dump(wallet_dump)
+
+        self.do_tool_createfromdump("native_endian", "endian.dump", "bdb")
+        native_dump = self.read_dump(self.nodes[0].datadir_path / "rt-native_endian.dump")
+        self.assert_dump(expected_dump, native_dump)
+
+        self.do_tool_createfromdump("other_endian", "endian.dump", "bdb_swap")
+        other_dump = self.read_dump(self.nodes[0].datadir_path / "rt-other_endian.dump")
+        self.assert_dump(expected_dump, other_dump)
+
+    def test_dump_very_large_records(self):
+        self.log.info("Test that wallets with large records are successfully dumped")
+
+        self.start_node(0)
+        self.nodes[0].createwallet("bigrecords")
+        wallet = self.nodes[0].get_wallet_rpc("bigrecords")
+
+        # Both BDB and sqlite have maximum page sizes of 65536 bytes, with defaults of 4096
+        # When a record exceeds some size threshold, both BDB and SQLite will store the data
+        # in one or more overflow pages. We want to make sure that our tooling can dump such
+        # records, even when they span multiple pages. To make a large record, we just need
+        # to make a very big transaction.
+        self.generate(self.nodes[0], 101)
+        def_wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name)
+        outputs = {}
+        for i in range(500):
+            outputs[wallet.getnewaddress(address_type="p2sh-segwit")] = 0.01
+        def_wallet.sendmany(amounts=outputs)
+        self.generate(self.nodes[0], 1)
+        send_res = wallet.sendall([def_wallet.getnewaddress()])
+        self.generate(self.nodes[0], 1)
+        assert_equal(send_res["complete"], True)
+        tx = wallet.gettransaction(txid=send_res["txid"], verbose=True)
+        assert_greater_than(tx["decoded"]["size"], 70000)
+
+        self.stop_node(0)
+
+        wallet_dump = self.nodes[0].datadir_path / "bigrecords.dump"
+        self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=bigrecords", f"-dumpfile={wallet_dump}", "dump")
+        dump = self.read_dump(wallet_dump)
+        for k,v in dump.items():
+            if tx["hex"] in v:
+                break
+        else:
+            assert False, "Big transaction was not found in wallet dump"
+
+    def test_dump_unclean_lsns(self):
+        if not self.options.bdbro:
+            return
+        self.log.info("Test that a legacy wallet that has not been compacted is not dumped by bdbro")
+
+        self.start_node(0, extra_args=["-flushwallet=0"])
+        self.nodes[0].createwallet("unclean_lsn")
+        wallet = self.nodes[0].get_wallet_rpc("unclean_lsn")
+        # First unload and load normally to make sure everything is written
+        wallet.unloadwallet()
+        self.nodes[0].loadwallet("unclean_lsn")
+        # Next cause a bunch of writes by filling the keypool
+        wallet.keypoolrefill(wallet.getwalletinfo()["keypoolsize"] + 100)
+        # Lastly kill bitcoind so that the LSNs don't get reset
+        self.nodes[0].process.kill()
+        self.nodes[0].wait_until_stopped(expected_ret_code=1 if platform.system() == "Windows" else -9)
+        assert self.nodes[0].is_node_stopped()
+
+        wallet_dump = self.nodes[0].datadir_path / "unclean_lsn.dump"
+        self.assert_raises_tool_error("LSNs are not reset, this database is not completely flushed. Please reopen then close the database with a version that has BDB support", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump")
+
+        # File can be dumped after reload it normally
+        self.start_node(0)
+        self.nodes[0].loadwallet("unclean_lsn")
+        self.stop_node(0)
+        self.assert_tool_output("The dumpfile may contain private keys. To ensure the safety of your Bitcoin, do not share the dumpfile.\n", "-wallet=unclean_lsn", f"-dumpfile={wallet_dump}", "dump")
+
     def run_test(self):
         self.wallet_path = self.nodes[0].wallets_path / self.default_wallet_name / self.wallet_data_filename
         self.test_invalid_tool_commands_and_args()
@@ -462,8 +556,11 @@ class ToolWalletTest(BitcoinTestFramework):
         if not self.options.descriptors:
             # Salvage is a legacy wallet only thing
             self.test_salvage()
+            self.test_dump_endianness()
+            self.test_dump_unclean_lsns()
         self.test_dump_createfromdump()
         self.test_chainless_conflicts()
+        self.test_dump_very_large_records()
 
 if __name__ == '__main__':
     ToolWalletTest().main()