Merge #19145: Add hash_type MUHASH for gettxoutsetinfo

e987ae5a55 test: Add test for deterministic UTXO set hash results (Fabian Jahr)
6ccc8fc067 test: Add test for gettxoutsetinfo RPC with MuHash (Fabian Jahr)
0d3b2f643d rpc: Add hash_type MUHASH to gettxoutsetinfo (Fabian Jahr)
2474645f3b refactor: Separate hash and stats calculation in coinstats (Fabian Jahr)
a1fcceac69 refactor: Improve encapsulation between MuHash3072 and Num3072 (Fabian Jahr)

Pull request description:

  This is another Pr in the series PRs for Coinstatsindex (see overview in #18000). This PR adds the `hash_type` option `muhash` to `gettxoutsetinfo` through which the user can calculate the serialized muhash of the utxo set. This PR does not use the index yet.

ACKs for top commit:
  Sjors:
    tACK e987ae5
  achow101:
    ACK e987ae5a55
  jonatack:
    Tested re-ACK e987ae5a55 per `git diff 3506d90 e987ae5`, reviewed diff, debug built, ran gettxoutsetinfo -signet and help on this branch vs master, at height 23127 both returned `hash_serialized_2` of `2b72d65f3b6efb2311f58374ea2b939abf49684d44f4bafda45faa3b5452a454` and this branch returned `muhash` of `c9f1ff12d345ccf9939c6bbf087e6f7399b6115adee1569287e9c5c43dbb475c`
  ryanofsky:
    Code review ACK e987ae5a55. Looks very good. I left one suggestion to simplify code, but feel free to ignore it here and maybe consider it for later since PR has already had a lot of review.

Tree-SHA512: 9a739ce375e73749fa69a467262b60d3e5314ef384e2d7150b3bbc8e4125cd9fd1db95306623bb9a632fcbaf5d9d2bf2f5cc43bf717d4ff5e2c9c4b52dd9296c

test/functional/feature_utxo_set_hash.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020-2021 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test UTXO set hash value calculation in gettxoutsetinfo."""
+
+import struct
+
+from test_framework.blocktools import create_transaction
+from test_framework.messages import (
+    CBlock,
+    COutPoint,
+    FromHex,
+)
+from test_framework.muhash import MuHash3072
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+class UTXOSetHashTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.setup_clean_chain = True
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def test_deterministic_hash_results(self):
+        self.log.info("Test deterministic UTXO set hash results")
+
+        # These depend on the setup_clean_chain option, the chain loaded from the cache
+        assert_equal(self.nodes[0].gettxoutsetinfo()['hash_serialized_2'], "b32ec1dda5a53cd025b95387aad344a801825fe46a60ff952ce26528f01d3be8")
+        assert_equal(self.nodes[0].gettxoutsetinfo("muhash")['muhash'], "dd5ad2a105c2d29495f577245c357409002329b9f4d6182c0af3dc2f462555c8")
+
+    def test_muhash_implementation(self):
+        self.log.info("Test MuHash implementation consistency")
+
+        node = self.nodes[0]
+
+        # Generate 100 blocks and remove the first since we plan to spend its
+        # coinbase
+        block_hashes = node.generate(100)
+        blocks = list(map(lambda block: FromHex(CBlock(), node.getblock(block, False)), block_hashes))
+        spending = blocks.pop(0)
+
+        # Create a spending transaction and mine a block which includes it
+        tx = create_transaction(node, spending.vtx[0].rehash(), node.getnewaddress(), amount=49)
+        txid = node.sendrawtransaction(hexstring=tx.serialize_with_witness().hex(), maxfeerate=0)
+
+        tx_block = node.generateblock(output=node.getnewaddress(), transactions=[txid])
+        blocks.append(FromHex(CBlock(), node.getblock(tx_block['hash'], False)))
+
+        # Serialize the outputs that should be in the UTXO set and add them to
+        # a MuHash object
+        muhash = MuHash3072()
+
+        for height, block in enumerate(blocks):
+            # The Genesis block coinbase is not part of the UTXO set and we
+            # spent the first mined block
+            height += 2
+
+            for tx in block.vtx:
+                for n, tx_out in enumerate(tx.vout):
+                    coinbase = 1 if not tx.vin[0].prevout.hash else 0
+
+                    # Skip witness commitment
+                    if (coinbase and n > 0):
+                        continue
+
+                    data = COutPoint(int(tx.rehash(), 16), n).serialize()
+                    data += struct.pack("<i", height * 2 + coinbase)
+                    data += tx_out.serialize()
+
+                    muhash.insert(data)
+
+        finalized = muhash.digest()
+        node_muhash = node.gettxoutsetinfo("muhash")['muhash']
+
+        assert_equal(finalized[::-1].hex(), node_muhash)
+
+    def run_test(self):
+        self.test_deterministic_hash_results()
+        self.test_muhash_implementation()
+
+
+if __name__ == '__main__':
+    UTXOSetHashTest().main()

test/functional/rpc_blockchain.py

@@ -268,6 +268,18 @@ class BlockchainTest(BitcoinTestFramework):
         res5 = node.gettxoutsetinfo(hash_type='none')
         assert 'hash_serialized_2' not in res5
 
+        # hash_type muhash should return a different UTXO set hash.
+        res6 = node.gettxoutsetinfo(hash_type='muhash')
+        assert 'muhash' in res6
+        assert(res['hash_serialized_2'] != res6['muhash'])
+
+        # muhash should not be included in gettxoutset unless requested.
+        for r in [res, res2, res3, res4, res5]:
+            assert 'muhash' not in r
+
+        # Unknown hash_type raises an error
+        assert_raises_rpc_error(-8, "foohash is not a valid hash_type", node.gettxoutsetinfo, "foohash")
+
     def _test_getblockheader(self):
         node = self.nodes[0]
 

test/functional/test_runner.py

@@ -203,6 +203,7 @@ BASE_SCRIPTS = [
     'feature_notifications.py',
     'rpc_getblockfilter.py',
     'rpc_invalidateblock.py',
+    'feature_utxo_set_hash.py',
     'feature_rbf.py',
     'mempool_packages.py',
     'mempool_package_onemore.py',