Merge bitcoin/bitcoin#26886: test: add rescan utxos inside MiniWallet's initialization

6bd098a838 test: simplify tests by using the pre-mined chain (kouloumos)
42029a7fd4 test: remove redundant blocks generation logic (kouloumos)
0377d6bb42 test: add `rescan_utxos` in MiniWallet's initialization (kouloumos)

Pull request description:

  When a pre-mined blockchain is used (default behavior), it [contains coinbase outputs in blocks 76-10](07c54de550/test/functional/test_framework/test_framework.py (L809-L813)) to [the MiniWallet's default address](07c54de550/test/functional/test_framework/wallet.py (L99-L101)). That's why we always* `rescan_utxos()` after initializing the MiniWallet, in order for the MiniWallet to account for those mature UTXOs.

  > The tests following this usage pattern can be seen with:
  > ```git grep -n "MiniWallet(" $(git grep -le "rescan_utxos()" $(git grep -Le "self.setup_clean_chain = True"))```

  **This PR adds `rescan_utxos()` inside MiniWallet's initialization to simplify usage when the MiniWallet is used with a pre-mined chain.**

  ### secondary changes

  - *There are a few tests that use the pre-mined blockchain but do not `rescan_utxos()`, they instead generate new blocks to create mature UTXOs.

    > Those were written before the `rescan_utxos()` method was introduced with https://github.com/bitcoin/bitcoin/pull/22955 (fac66d0a39) and can be seen with:
    > `git grep -n "MiniWallet(" $(git grep -Le "rescan_utxos()" $(git grep -Le "self.setup_clean_chain = True"))`
    >

    After including `rescan_utxos()` inside MiniWallets initilization, this blocks generation logic is not needed as the MiniWallet already accounts for enough mature UTXOs to perform the tests. **Therefore the now redundant blocks generation logic is removed from those tests with the second commit.**

  - The rest of the MiniWallet tests use a clean chain (`self.setup_clean_chain = True`)  and can be seen with
    `git grep -n "MiniWallet(" $(git grep -le "self.setup_clean_chain = True")`

    From those, there are a few that start from a clean chain and then create enough mature UTXOs for the MiniWallet with this kind of logic:
   07c54de550/test/functional/mempool_expiry.py (L36-L40)

    **Those tests are simplified in the third commit to instead utilize the mature UTXOs of the pre-mined chain.**

ACKs for top commit:
  MarcoFalke:
    ACK 6bd098a838 🕷
  theStack:
    re-ACK 6bd098a838

Tree-SHA512: 7f9361e36910e4000c33a32efdde4449f4a8a763bb42df96de826fcde469f9362f701b8c99e2a2c482d2d5a42a83ae5ae3844fdbed187ed0ff231f386c222493

test/functional/feature_bip68_sequence.py

@@ -63,9 +63,6 @@ class BIP68Test(BitcoinTestFramework):
         self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
         self.wallet = MiniWallet(self.nodes[0])
 
-        # Generate some coins
-        self.generate(self.wallet, 110)
-
         self.log.info("Running test disable flag")
         self.test_disable_flag()
 

test/functional/feature_dbcrash.py

@@ -202,7 +202,6 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[3])
-        self.wallet.rescan_utxos()
         initial_height = self.nodes[3].getblockcount()
         self.generate(self.nodes[3], COINBASE_MATURITY, sync_fun=self.no_op)
 

test/functional/feature_fee_estimation.py

@@ -297,7 +297,6 @@ class EstimateFeeTest(BitcoinTestFramework):
         # Split two coinbases into many small utxos
         self.start_node(0)
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
         self.initial_split(self.nodes[0])
         self.log.info("Finished splitting")
 

test/functional/feature_rbf.py

@@ -42,10 +42,6 @@ class ReplaceByFeeTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        # the pre-mined test framework chain contains coinbase outputs to the
-        # MiniWallet's default address in blocks 76-100 (see method
-        # BitcoinTestFramework._initialize_chain())
-        self.wallet.rescan_utxos()
 
         self.log.info("Running test simple doublespend...")
         self.test_simple_doublespend()
@@ -398,7 +394,6 @@ class ReplaceByFeeTest(BitcoinTestFramework):
         """
         normal_node = self.nodes[1]
         wallet = MiniWallet(normal_node)
-        wallet.rescan_utxos()
         # Clear mempools to avoid cross-node sync failure.
         for node in self.nodes:
             self.generate(node, 1)

test/functional/feature_txindex_compatibility.py

@@ -42,7 +42,6 @@ class TxindexCompatibilityTest(BitcoinTestFramework):
 
     def run_test(self):
         mini_wallet = MiniWallet(self.nodes[1])
-        mini_wallet.rescan_utxos()
         spend_utxo = mini_wallet.get_utxo()
         mini_wallet.send_self_transfer(from_node=self.nodes[1], utxo_to_spend=spend_utxo)
         self.generate(self.nodes[1], 1)

test/functional/interface_rest.py

@@ -96,7 +96,6 @@ class RESTTest (BitcoinTestFramework):
     def run_test(self):
         self.url = urllib.parse.urlparse(self.nodes[0].url)
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         self.log.info("Broadcast test transaction and sync nodes")
         txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=getnewdestination()[1], amount=int(0.1 * COIN))

test/functional/interface_usdt_utxocache.py

@@ -144,7 +144,6 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.generate(self.wallet, 101)
 
         self.test_uncache()
         self.test_add_spent()

test/functional/interface_zmq.py

@@ -215,7 +215,6 @@ class ZMQTest (BitcoinTestFramework):
             assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"])
 
 
-        self.wallet.rescan_utxos()
         self.log.info("Wait for tx from second node")
         payment_tx = self.wallet.send_self_transfer(from_node=self.nodes[1])
         payment_txid = payment_tx['txid']

test/functional/mempool_accept.py

@@ -69,7 +69,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework):
     def run_test(self):
         node = self.nodes[0]
         self.wallet = MiniWallet(node)
-        self.wallet.rescan_utxos()
 
         self.log.info('Start with empty mempool, and 200 blocks')
         self.mempool_size = 0

test/functional/mempool_datacarrier.py

@@ -44,7 +44,6 @@ class DataCarrierTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         # By default, only 80 bytes are used for data (+1 for OP_RETURN, +2 for the pushdata opcodes).
         default_size_data = random_bytes(MAX_OP_RETURN_RELAY - 3)

test/functional/mempool_dust.py

@@ -72,7 +72,6 @@ class DustRelayFeeTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         # prepare output scripts of each standard type
         key = ECKey()

test/functional/mempool_expiry.py

@@ -12,7 +12,6 @@ definable expiry timeout via the '-mempoolexpiry=<n>' command line argument
 
 from datetime import timedelta
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.messages import DEFAULT_MEMPOOL_EXPIRY_HOURS
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
@@ -27,17 +26,11 @@ CUSTOM_MEMPOOL_EXPIRY = 10  # hours
 class MempoolExpiryTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-        self.setup_clean_chain = True
 
     def test_transaction_expiry(self, timeout):
         """Tests that a transaction expires after the expiry timeout and its
         children are removed as well."""
         node = self.nodes[0]
-        self.wallet = MiniWallet(node)
-
-        # Add enough mature utxos to the wallet so that all txs spend confirmed coins.
-        self.generate(self.wallet, 4)
-        self.generate(node, COINBASE_MATURITY)
 
         # Send a parent transaction that will expire.
         parent_txid = self.wallet.send_self_transfer(from_node=node)['txid']
@@ -97,6 +90,8 @@ class MempoolExpiryTest(BitcoinTestFramework):
         assert_equal(half_expiry_time, node.getmempoolentry(independent_txid)['time'])
 
     def run_test(self):
+        self.wallet = MiniWallet(self.nodes[0])
+
         self.log.info('Test default mempool expiry timeout of %d hours.' %
                       DEFAULT_MEMPOOL_EXPIRY_HOURS)
         self.test_transaction_expiry(DEFAULT_MEMPOOL_EXPIRY_HOURS)

test/functional/mempool_package_onemore.py

@@ -31,7 +31,6 @@ class MempoolPackagesTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         # DEFAULT_ANCESTOR_LIMIT transactions off a confirmed tx should be fine
         chain = []

test/functional/mempool_persist.py

@@ -59,7 +59,6 @@ class MempoolPersistTest(BitcoinTestFramework):
 
     def run_test(self):
         self.mini_wallet = MiniWallet(self.nodes[2])
-        self.mini_wallet.rescan_utxos()
         if self.is_sqlite_compiled():
             self.nodes[2].createwallet(
                 wallet_name="watch",

test/functional/mempool_reorg.py

@@ -31,7 +31,6 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         self.log.info("Add 4 coinbase utxos to the miniwallet")
         # Block 76 contains the first spendable coinbase txs.
         first_block = 76
-        wallet.rescan_utxos()
 
         # Three scenarios for re-orging coinbase spends in the memory pool:
         # 1. Direct coinbase spend  :  spend_1

test/functional/mempool_resurrect.py

@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test resurrection of mined transactions when the blockchain is re-organized."""
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import assert_equal
 from test_framework.wallet import MiniWallet
@@ -13,16 +12,11 @@ from test_framework.wallet import MiniWallet
 class MempoolCoinbaseTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-        self.setup_clean_chain = True
 
     def run_test(self):
         node = self.nodes[0]
         wallet = MiniWallet(node)
 
-        # Add enough mature utxos to the wallet so that all txs spend confirmed coins
-        self.generate(wallet, 3)
-        self.generate(node, COINBASE_MATURITY)
-
         # Spend block 1/2/3's coinbase transactions
         # Mine a block
         # Create three more transactions, spending the spends

test/functional/mempool_spend_coinbase.py

@@ -28,7 +28,6 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework):
         chain_height = 198
         self.nodes[0].invalidateblock(self.nodes[0].getblockhash(chain_height + 1))
         assert_equal(chain_height, self.nodes[0].getblockcount())
-        wallet.rescan_utxos()
 
         # Coinbase at height chain_height-100+1 ok in mempool, should
         # get mined. Coinbase at height chain_height-100+2 is

test/functional/mempool_unbroadcast.py

@@ -23,7 +23,6 @@ class MempoolUnbroadcastTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
         self.test_broadcast()
         self.test_txn_removal()
 

test/functional/mining_getblocktemplate_longpoll.py

@@ -8,7 +8,6 @@ from decimal import Decimal
 import random
 import threading
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import get_rpc_proxy
 from test_framework.wallet import MiniWallet
@@ -62,9 +61,6 @@ class GetBlockTemplateLPTest(BitcoinTestFramework):
         thr.join(5)  # wait 5 seconds or until thread exits
         assert not thr.is_alive()
 
-        # Add enough mature utxos to the wallets, so that all txs spend confirmed coins
-        self.generate(self.nodes[0], COINBASE_MATURITY)
-
         self.log.info("Test that introducing a new transaction into the mempool will terminate the longpoll")
         thr = LongpollThread(self.nodes[0])
         thr.start()

test/functional/mining_prioritisetransaction.py

@@ -106,7 +106,6 @@ class PrioritiseTransactionTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         # Test `prioritisetransaction` required parameters
         assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction)

test/functional/p2p_blocksonly.py

@@ -20,8 +20,6 @@ class P2PBlocksOnly(BitcoinTestFramework):
 
     def run_test(self):
         self.miniwallet = MiniWallet(self.nodes[0])
-        # Add enough mature utxos to the wallet, so that all txs spend confirmed coins
-        self.miniwallet.rescan_utxos()
 
         self.blocksonly_mode_tests()
         self.blocks_relay_conn_tests()

test/functional/p2p_feefilter.py

@@ -6,7 +6,6 @@
 
 from decimal import Decimal
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter
 from test_framework.p2p import P2PInterface, p2p_lock
 from test_framework.test_framework import BitcoinTestFramework
@@ -80,9 +79,6 @@ class FeeFilterTest(BitcoinTestFramework):
         node1 = self.nodes[1]
         node0 = self.nodes[0]
         miniwallet = MiniWallet(node1)
-        # Add enough mature utxos to the wallet, so that all txs spend confirmed coins
-        self.generate(miniwallet, 5)
-        self.generate(node1, COINBASE_MATURITY)
 
         conn = self.nodes[0].add_p2p_connection(TestP2PConn())
 

test/functional/p2p_filter.py

@@ -214,7 +214,6 @@ class FilterTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         filter_peer = self.nodes[0].add_p2p_connection(P2PBloomFilter())
         self.log.info('Test filter size limits')

test/functional/p2p_leak_tx.py

@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test that we don't leak txs to inbound peers that we haven't yet announced to"""
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.messages import msg_getdata, CInv, MSG_TX
 from test_framework.p2p import p2p_lock, P2PDataStore
 from test_framework.test_framework import BitcoinTestFramework
@@ -26,9 +25,6 @@ class P2PLeakTxTest(BitcoinTestFramework):
     def run_test(self):
         gen_node = self.nodes[0]  # The block and tx generating node
         miniwallet = MiniWallet(gen_node)
-        # Add enough mature utxos to the wallet, so that all txs spend confirmed coins
-        self.generate(miniwallet, 1)
-        self.generate(gen_node, COINBASE_MATURITY)
 
         inbound_peer = self.nodes[0].add_p2p_connection(P2PNode())  # An "attacking" inbound peer
 

test/functional/p2p_permissions.py

@@ -26,7 +26,6 @@ class P2PPermissionsTests(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         self.check_tx_relay()
 

test/functional/p2p_tx_privacy.py

@@ -53,7 +53,6 @@ class TxPrivacyTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         tx_originator = self.nodes[0].add_p2p_connection(P2PInterface())
         spy = self.nodes[0].add_p2p_connection(P2PTxSpy(), wait_for_verack=False)

test/functional/rpc_generate.py

@@ -28,7 +28,6 @@ class RPCGenerateTest(BitcoinTestFramework):
     def test_generateblock(self):
         node = self.nodes[0]
         miniwallet = MiniWallet(node)
-        miniwallet.rescan_utxos()
 
         self.log.info('Generate an empty block to address')
         address = miniwallet.get_address()

test/functional/rpc_mempool_info.py

@@ -18,7 +18,6 @@ class RPCMempoolInfoTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
         confirmed_utxo = self.wallet.get_utxo()
 
         # Create a tree of unconfirmed transactions in the mempool:

test/functional/rpc_net.py

@@ -11,7 +11,6 @@ from decimal import Decimal
 from itertools import product
 import time
 
-from test_framework.blocktools import COINBASE_MATURITY
 import test_framework.messages
 from test_framework.p2p import (
     P2PInterface,
@@ -43,7 +42,6 @@ def assert_net_servicesnames(servicesflag, servicenames):
 
 class NetTest(BitcoinTestFramework):
     def set_test_params(self):
-        self.setup_clean_chain = True
         self.num_nodes = 2
         self.extra_args = [["-minrelaytxfee=0.00001000"], ["-minrelaytxfee=0.00000500"]]
         self.supports_cli = False
@@ -51,9 +49,6 @@ class NetTest(BitcoinTestFramework):
     def run_test(self):
         # We need miniwallet to make a transaction
         self.wallet = MiniWallet(self.nodes[0])
-        self.generate(self.wallet, 1)
-        # Get out of IBD for the minfeefilter and getpeerinfo tests.
-        self.generate(self.nodes[0], COINBASE_MATURITY + 1)
 
         # By default, the test framework sets up an addnode connection from
         # node 1 --> node0. By connecting node0 --> node 1, we're left with

test/functional/rpc_rawtransaction.py

@@ -16,7 +16,6 @@ from collections import OrderedDict
 from decimal import Decimal
 from itertools import product
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.messages import (
     MAX_BIP125_RBF_SEQUENCE,
     CTransaction,
@@ -59,7 +58,6 @@ class RawTransactionsTest(BitcoinTestFramework):
         self.add_wallet_options(parser, descriptors=False)
 
     def set_test_params(self):
-        self.setup_clean_chain = True
         self.num_nodes = 3
         self.extra_args = [
             ["-txindex"],
@@ -77,9 +75,6 @@ class RawTransactionsTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.log.info("Prepare some coins for multiple *rawtransaction commands")
-        self.generate(self.wallet, 10)
-        self.generate(self.nodes[0], COINBASE_MATURITY + 1)
 
         self.getrawtransaction_tests()
         self.getrawtransaction_verbosity_tests()

test/functional/rpc_scanblocks.py

@@ -27,7 +27,6 @@ class ScanblocksTest(BitcoinTestFramework):
     def run_test(self):
         node = self.nodes[0]
         wallet = MiniWallet(node)
-        wallet.rescan_utxos()
 
         # send 1.0, mempool only
         _, spk_1, addr_1 = getnewdestination()

test/functional/rpc_scantxoutset.py

@@ -31,7 +31,6 @@ class ScantxoutsetTest(BitcoinTestFramework):
 
     def run_test(self):
         self.wallet = MiniWallet(self.nodes[0])
-        self.wallet.rescan_utxos()
 
         self.log.info("Test if we find coinbase outputs.")
         assert_equal(sum(u["coinbase"] for u in self.nodes[0].scantxoutset("start", [self.wallet.get_descriptor()])["unspents"]), 49)

test/functional/rpc_txoutproof.py

@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test gettxoutproof and verifytxoutproof RPCs."""
 
-from test_framework.blocktools import COINBASE_MATURITY
 from test_framework.messages import (
     CMerkleBlock,
     from_hex,
@@ -20,7 +19,6 @@ from test_framework.wallet import MiniWallet
 class MerkleBlockTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
-        self.setup_clean_chain = True
         self.extra_args = [
             [],
             ["-txindex"],
@@ -28,12 +26,9 @@ class MerkleBlockTest(BitcoinTestFramework):
 
     def run_test(self):
         miniwallet = MiniWallet(self.nodes[0])
-        # Add enough mature utxos to the wallet, so that all txs spend confirmed coins
-        self.generate(miniwallet, 5)
-        self.generate(self.nodes[0], COINBASE_MATURITY)
 
         chain_height = self.nodes[1].getblockcount()
-        assert_equal(chain_height, 105)
+        assert_equal(chain_height, 200)
 
         txid1 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid']
         txid2 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid']

test/functional/test_framework/wallet.py

@@ -101,6 +101,13 @@ class MiniWallet:
             self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true()
             self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey'])
 
+        # When the pre-mined test framework chain is used, it contains coinbase
+        # outputs to the MiniWallet's default address in blocks 76-100
+        # (see method BitcoinTestFramework._initialize_chain())
+        # The MiniWallet needs to rescan_utxos() in order to account
+        # for those mature UTXOs, so that all txs spend confirmed coins
+        self.rescan_utxos()
+
     def _create_utxo(self, *, txid, vout, value, height, coinbase, confirmations):
         return {"txid": txid, "vout": vout, "value": value, "height": height, "coinbase": coinbase, "confirmations": confirmations}
 

test/functional/wallet_fast_rescan.py

@@ -40,7 +40,6 @@ class WalletFastRescanTest(BitcoinTestFramework):
     def run_test(self):
         node = self.nodes[0]
         wallet = MiniWallet(node)
-        wallet.rescan_utxos()
 
         self.log.info("Create descriptor wallet with backup")
         WALLET_BACKUP_FILENAME = os.path.join(node.datadir, 'wallet.bak')