rpc: faster getblockstats using BlockUndo data

Using undo data for a block (rev?????.dat) we can retrieve value information about prevouts and calculate the final transaction fee (rate). This approach is about 80x faster, drops the requirement for -txindex, and works for all non-pruned blocks.

test/functional/rpc_getblockstats.py

@@ -13,7 +13,6 @@ from test_framework.util import (
 )
 import json
 import os
-import time
 
 TESTSDIR = os.path.dirname(os.path.realpath(__file__))
 
@@ -21,18 +20,6 @@ class GetblockstatsTest(BitcoinTestFramework):
 
     start_height = 101
     max_stat_pos = 2
-    STATS_NEED_TXINDEX = [
-        'avgfee',
-        'avgfeerate',
-        'maxfee',
-        'maxfeerate',
-        'medianfee',
-        'feerate_percentiles',
-        'minfee',
-        'minfeerate',
-        'totalfee',
-        'utxo_size_inc',
-    ]
 
     def add_options(self, parser):
         parser.add_argument('--gen-test-data', dest='gen_test_data',
@@ -44,24 +31,26 @@ class GetblockstatsTest(BitcoinTestFramework):
                             help='Test data file')
 
     def set_test_params(self):
-        self.num_nodes = 2
-        self.extra_args = [['-txindex'], ['-paytxfee=0.003']]
+        self.num_nodes = 1
         self.setup_clean_chain = True
 
     def get_stats(self):
         return [self.nodes[0].getblockstats(hash_or_height=self.start_height + i) for i in range(self.max_stat_pos+1)]
 
     def generate_test_data(self, filename):
-        mocktime = time.time()
+        mocktime = 1525107225
+        self.nodes[0].setmocktime(mocktime)
         self.nodes[0].generate(101)
 
-        self.nodes[0].sendtoaddress(address=self.nodes[1].getnewaddress(), amount=10, subtractfeefromamount=True)
+        address = self.nodes[0].get_deterministic_priv_key().address
+        self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=True)
         self.nodes[0].generate(1)
         self.sync_all()
 
-        self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=10, subtractfeefromamount=True)
-        self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=10, subtractfeefromamount=False)
-        self.nodes[1].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1, subtractfeefromamount=True)
+        self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=True)
+        self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=False)
+        self.nodes[0].settxfee(amount=0.003)
+        self.nodes[0].sendtoaddress(address=address, amount=1, subtractfeefromamount=True)
         self.sync_all()
         self.nodes[0].generate(1)
 
@@ -93,11 +82,12 @@ class GetblockstatsTest(BitcoinTestFramework):
 
         # Set the timestamps from the file so that the nodes can get out of Initial Block Download
         self.nodes[0].setmocktime(mocktime)
-        self.nodes[1].setmocktime(mocktime)
+        self.sync_all()
 
         for b in blocks:
             self.nodes[0].submitblock(b)
 
+
     def run_test(self):
         test_data = os.path.join(TESTSDIR, self.options.test_data)
         if self.options.gen_test_data:
@@ -107,9 +97,6 @@ class GetblockstatsTest(BitcoinTestFramework):
 
         self.sync_all()
         stats = self.get_stats()
-        expected_stats_noindex = []
-        for stat_row in stats:
-            expected_stats_noindex.append({k: v for k, v in stat_row.items() if k not in self.STATS_NEED_TXINDEX})
 
         # Make sure all valid statistics are included but nothing else is
         expected_keys = self.expected_stats[0].keys()
@@ -127,10 +114,6 @@ class GetblockstatsTest(BitcoinTestFramework):
             stats_by_hash = self.nodes[0].getblockstats(hash_or_height=blockhash)
             assert_equal(stats_by_hash, self.expected_stats[i])
 
-            # Check with the node that has no txindex
-            stats_no_txindex = self.nodes[1].getblockstats(hash_or_height=blockhash, stats=list(expected_stats_noindex[i].keys()))
-            assert_equal(stats_no_txindex, expected_stats_noindex[i])
-
         # Make sure each stat can be queried on its own
         for stat in expected_keys:
             for i in range(self.max_stat_pos+1):
@@ -168,12 +151,6 @@ class GetblockstatsTest(BitcoinTestFramework):
         # Make sure we aren't always returning inv_sel_stat as the culprit stat
         assert_raises_rpc_error(-8, 'Invalid selected statistic aaa%s' % inv_sel_stat,
                                 self.nodes[0].getblockstats, hash_or_height=1, stats=['minfee' , 'aaa%s' % inv_sel_stat])
-
-        assert_raises_rpc_error(-8, 'One or more of the selected stats requires -txindex enabled',
-                                self.nodes[1].getblockstats, hash_or_height=1)
-        assert_raises_rpc_error(-8, 'One or more of the selected stats requires -txindex enabled',
-                                self.nodes[1].getblockstats, hash_or_height=self.start_height + self.max_stat_pos)
-
         # Mainchain's genesis block shouldn't be found on regtest
         assert_raises_rpc_error(-5, 'Block not found', self.nodes[0].getblockstats,
                                 hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')