Merge bitcoin/bitcoin#30221: wallet: Ensure best block matches wallet scan state

30a94b1ab9 test, wallet: Remove concurrent writes test (Ava Chow)
b44b7c03fe wallet: Write best block record on unload (Ava Chow)
876a2585a8 wallet: Remove unnecessary database Close step on shutdown (Ava Chow)
98a1a5275c wallet: Remove chainStateFlushed (Ava Chow)
7fd3e1cf0c wallet, bench: Write a bestblock record in WalletMigration (Ava Chow)
6d3a8b195a wallet: Replace chainStateFlushed in loading with SetLastBlockProcessed (Ava Chow)
7bacabb204 wallet: Update best block record after block dis/connect (Ava Chow)

Pull request description:

  Implements the idea discussed in https://github.com/bitcoin/bitcoin/pull/29652#issuecomment-2010579484

  Currently, `m_last_block_processed` and `m_last_block_processed_height` are not guaranteed to match the block locator stored in the wallet, nor do either of those fields actually represent the last block that the wallet is synced up to. This is confusing and unintuitive.

  This PR changes those last block fields to be updated whenever the wallet makes a change to the db for new transaction state found in new blocks. Whenever a block is received that contains a transaction relevant to the wallet, the last block locator will now be written to disk. Furthermore, every block disconnection will now write an updated locator.

  To ensure that the locator is relatively recent and loading rescans are fairly quick in the event of unplanned shutdown, it is also now written every 144 blocks (~1 day). Additionally it is now written when the wallet is unloaded so that it is accurate when the wallet is loaded again.

  Lastly, the `chainstateFlushed` notification in the wallet is changed to be a no-op. The best block locator record is no longer written when `chainstateFlushed` is received from the node since it should already be mostly up to date.

ACKs for top commit:
  rkrux:
    ACK 30a94b1ab9
  mzumsande:
    Code Review ACK 30a94b1ab9
  ryanofsky:
    Code review ACK 30a94b1ab9. Only changes since last review are using WriteBestBlock method more places and updating comments.

Tree-SHA512: 46117541f8aaf13dde57430e813b4bbbd5e146e2632769675803c8e65a82f149a7cc6026489a127d32684b90124bd2b7c28216dbcfa6a47447300e8f3814e029

test/functional/wallet_descriptor.py

@@ -9,10 +9,7 @@ try:
 except ImportError:
     pass
 
-import concurrent.futures
-
 from test_framework.blocktools import COINBASE_MATURITY
-from test_framework.descriptors import descsum_create
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_not_equal,
@@ -33,41 +30,6 @@ class WalletDescriptorTest(BitcoinTestFramework):
         self.skip_if_no_wallet()
         self.skip_if_no_py_sqlite3()
 
-    def test_concurrent_writes(self):
-        self.log.info("Test sqlite concurrent writes are in the correct order")
-        self.restart_node(0, extra_args=["-unsafesqlitesync=0"])
-        self.nodes[0].createwallet(wallet_name="concurrency", blank=True)
-        wallet = self.nodes[0].get_wallet_rpc("concurrency")
-        # First import a descriptor that uses hardened dervation so that topping up
-        # Will require writing a ton to db
-        wallet.importdescriptors([{"desc":descsum_create("wpkh(tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg/0h/0h/*h)"), "timestamp": "now", "active": True}])
-        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as thread:
-            topup = thread.submit(wallet.keypoolrefill, newsize=1000)
-
-            # Then while the topup is running, we need to do something that will call
-            # ChainStateFlushed which will trigger a write to the db, hopefully at the
-            # same time that the topup still has an open db transaction.
-            self.nodes[0].cli.gettxoutsetinfo()
-            assert_equal(topup.result(), None)
-
-        wallet.unloadwallet()
-
-        # Check that everything was written
-        wallet_db = self.nodes[0].wallets_path / "concurrency" / self.wallet_data_filename
-        conn = sqlite3.connect(wallet_db)
-        with conn:
-            # Retrieve the bestblock_nomerkle record
-            bestblock_rec = conn.execute("SELECT value FROM main WHERE hex(key) = '1262657374626C6F636B5F6E6F6D65726B6C65'").fetchone()[0]
-            # Retrieve the number of descriptor cache records
-            # Since we store binary data, sqlite's comparison operators don't work everywhere
-            # so just retrieve all records and process them ourselves.
-            db_keys = conn.execute("SELECT key FROM main").fetchall()
-            cache_records = len([k[0] for k in db_keys if b"walletdescriptorcache" in k[0]])
-        conn.close()
-
-        assert_equal(bestblock_rec[5:37][::-1].hex(), self.nodes[0].getbestblockhash())
-        assert_equal(cache_records, 1000)
-
     def run_test(self):
         # Make a descriptor wallet
         self.log.info("Making a descriptor wallet")
@@ -254,8 +216,6 @@ class WalletDescriptorTest(BitcoinTestFramework):
         conn.close()
         assert_raises_rpc_error(-4, "Unexpected legacy entry in descriptor wallet found.", self.nodes[0].loadwallet, "crashme")
 
-        self.test_concurrent_writes()
-
 
 if __name__ == '__main__':
     WalletDescriptorTest(__file__).main()

test/functional/wallet_reorgsrestore.py

@@ -119,11 +119,11 @@ class ReorgsRestoreTest(BitcoinTestFramework):
         self.start_node(0)
         assert_equal(node.getbestblockhash(), tip)
 
-        # Due to an existing bug, the wallet incorrectly keeps the transaction in an abandoned state, even though that's
-        # no longer the case (after the unclean shutdown, the node's chain returned to the pre-invalidation tip).
-        # This issue blocks any future spending and results in an incorrect balance display.
+        # After disconnecting the block, the wallet should record the new best block.
+        # Upon reload after the crash, since the chainstate was not flushed, the tip contains the previously abandoned
+        # coinbase. This should be rescanned and now un-abandoned.
         wallet = node.get_wallet_rpc("reorg_crash")
-        assert_equal(wallet.getwalletinfo()['immature_balance'], 0) # FIXME: #31824.
+        assert_equal(wallet.gettransaction(coinbase_tx_id)['details'][0]['abandoned'], False)
 
         # Previously, a bug caused the node to crash if two block disconnection events occurred consecutively.
         # Ensure this is no longer the case by simulating a new reorg.