Merge #17681: wallet: Keep inactive seeds after sethdseed and derive keys from them as needed

1ed52fbb4d Remove IBD check in sethdseed (Andrew Chow)
b1810a145a Test that keys from inactive seeds are generated (Andrew Chow)
c93082ece4 Generate new keys for inactive seeds after marking used (Andrew Chow)
45f2f6a0e8 Determine inactive HD seeds from key metadata and track them in LegacyScriptPubKeyMan (Andrew Chow)
b59b4504ab have GenerateNewKey and DeriveNewChildKey take a CHDChain as an argument (Andrew Chow)

Pull request description:

  Largely implements the suggestion from https://github.com/bitcoin/bitcoin/pull/17484#issuecomment-560845316.

  After `sethdseed` is called, the CHDChain for the old seed is kept in the wallet. It is kept on the file as a new `inactivehdseed` record and in memory in a map `m_inactive_hd_seeds`. In `LegacyScriptPubKeyMan::MarkUnusedAddresses` we check each used key's metadata for whether it was derived from an inactive seed. If it is, we then check to see how many keys after that key were derived from the inactive seed. If that number does not match the keypool parameter, we derive more keys from the inactive seed until it does match. This way we won't miss transactions belonging to keys outside of the range of the keypool initially.

  The indexes and internal-ness of a key is gotten by checking it's key origin data.

  Because of this change, we no longer need to wait for IBD to finish before `sethdseed` can work so that check is also removed.

  A test case for this is added as well which fails on master.

ACKs for top commit:
  ryanofsky:
    Code review ACK 1ed52fbb4d. Changes since last review: various commit message, code comment, log message, error checking improvements, and fix for topping up inactive seeds if wallet isn't reloaded after calling sethdseed and test for this
  ariard:
    Code Review ACK 1ed52fb
  jonatack:
    ACK 1ed52fbb4d thanks for addressing the previous review feedback; would be happy to see the new review questions answered and feedback addressed and re-ack.

Tree-SHA512: e658ae0e1dab94be55d2b62cdda506c94815e73a6881533fd30d41cc77477f82fee2095144957a3a1df0c129e256bdd7b7abe3737d515f393610446cae4edf1c

test/functional/wallet_hd.py

@@ -170,5 +170,101 @@ class WalletHDTest(BitcoinTestFramework):
             assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, new_seed)
             assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress()))
 
+            self.log.info('Test sethdseed restoring with keys outside of the initial keypool')
+            self.nodes[0].generate(10)
+            # Restart node 1 with keypool of 3 and a different wallet
+            self.nodes[1].createwallet(wallet_name='origin', blank=True)
+            self.stop_node(1)
+            self.start_node(1, extra_args=['-keypool=3', '-wallet=origin'])
+            connect_nodes(self.nodes[0], 1)
+
+            # sethdseed restoring and seeing txs to addresses out of the keypool
+            origin_rpc = self.nodes[1].get_wallet_rpc('origin')
+            seed = self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress())
+            origin_rpc.sethdseed(True, seed)
+
+            self.nodes[1].createwallet(wallet_name='restore', blank=True)
+            restore_rpc = self.nodes[1].get_wallet_rpc('restore')
+            restore_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc
+            restore_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive
+
+            self.nodes[1].createwallet(wallet_name='restore2', blank=True)
+            restore2_rpc = self.nodes[1].get_wallet_rpc('restore2')
+            restore2_rpc.sethdseed(True, seed) # Set to be the same seed as origin_rpc
+            restore2_rpc.sethdseed(True) # Rotate to a new seed, making original `seed` inactive
+
+            # Check persistence of inactive seed by reloading restore. restore2 is still loaded to test the case where the wallet is not reloaded
+            restore_rpc.unloadwallet()
+            self.nodes[1].loadwallet('restore')
+            restore_rpc = self.nodes[1].get_wallet_rpc('restore')
+
+            # Empty origin keypool and get an address that is beyond the initial keypool
+            origin_rpc.getnewaddress()
+            origin_rpc.getnewaddress()
+            last_addr = origin_rpc.getnewaddress() # Last address of initial keypool
+            addr = origin_rpc.getnewaddress() # First address beyond initial keypool
+
+            # Check that the restored seed has last_addr but does not have addr
+            info = restore_rpc.getaddressinfo(last_addr)
+            assert_equal(info['ismine'], True)
+            info = restore_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], False)
+            info = restore2_rpc.getaddressinfo(last_addr)
+            assert_equal(info['ismine'], True)
+            info = restore2_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], False)
+            # Check that the origin seed has addr
+            info = origin_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], True)
+
+            # Send a transaction to addr, which is out of the initial keypool.
+            # The wallet that has set a new seed (restore_rpc) should not detect this transaction.
+            txid = self.nodes[0].sendtoaddress(addr, 1)
+            origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex'])
+            self.nodes[0].generate(1)
+            origin_rpc.gettransaction(txid)
+            assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, txid)
+            out_of_kp_txid = txid
+
+            # Send a transaction to last_addr, which is in the initial keypool.
+            # The wallet that has set a new seed (restore_rpc) should detect this transaction and generate 3 new keys from the initial seed.
+            # The previous transaction (out_of_kp_txid) should still not be detected as a rescan is required.
+            txid = self.nodes[0].sendtoaddress(last_addr, 1)
+            origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex'])
+            self.nodes[0].generate(1)
+            origin_rpc.gettransaction(txid)
+            restore_rpc.gettransaction(txid)
+            assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, out_of_kp_txid)
+            restore2_rpc.gettransaction(txid)
+            assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore2_rpc.gettransaction, out_of_kp_txid)
+
+            # After rescanning, restore_rpc should now see out_of_kp_txid and generate an additional key.
+            # addr should now be part of restore_rpc and be ismine
+            restore_rpc.rescanblockchain()
+            restore_rpc.gettransaction(out_of_kp_txid)
+            info = restore_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], True)
+            restore2_rpc.rescanblockchain()
+            restore2_rpc.gettransaction(out_of_kp_txid)
+            info = restore2_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], True)
+
+            # Check again that 3 keys were derived.
+            # Empty keypool and get an address that is beyond the initial keypool
+            origin_rpc.getnewaddress()
+            origin_rpc.getnewaddress()
+            last_addr = origin_rpc.getnewaddress()
+            addr = origin_rpc.getnewaddress()
+
+            # Check that the restored seed has last_addr but does not have addr
+            info = restore_rpc.getaddressinfo(last_addr)
+            assert_equal(info['ismine'], True)
+            info = restore_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], False)
+            info = restore2_rpc.getaddressinfo(last_addr)
+            assert_equal(info['ismine'], True)
+            info = restore2_rpc.getaddressinfo(addr)
+            assert_equal(info['ismine'], False)
+
 if __name__ == '__main__':
     WalletHDTest().main ()