Merge bitcoin/bitcoin#25717: p2p: Implement anti-DoS headers sync

3add234546 ui: show header pre-synchronization progress (Pieter Wuille)
738421c50f Emit NotifyHeaderTip signals for pre-synchronization progress (Pieter Wuille)
376086fc5a Make validation interface capable of signalling header presync (Pieter Wuille)
93eae27031 Test large reorgs with headerssync logic (Suhas Daftuar)
355547334f Track headers presync progress and log it (Pieter Wuille)
03712dddfb Expose HeadersSyncState::m_current_height in getpeerinfo() (Suhas Daftuar)
150a5486db Test headers sync using minchainwork threshold (Suhas Daftuar)
0b6aa826b5 Add unit test for HeadersSyncState (Suhas Daftuar)
83c6a0c524 Reduce spurious messages during headers sync (Suhas Daftuar)
ed6cddd98e Require callers of AcceptBlockHeader() to perform anti-dos checks (Suhas Daftuar)
551a8d957c Utilize anti-DoS headers download strategy (Suhas Daftuar)
ed470940cd Add functions to construct locators without CChain (Pieter Wuille)
84852bb6bb Add bitdeque, an std::deque<bool> analogue that does bit packing. (Pieter Wuille)
1d4cfa4272 Add function to validate difficulty changes (Suhas Daftuar)

Pull request description:

  New nodes starting up for the first time lack protection against DoS from low-difficulty headers. While checkpoints serve as our protection against headers that fork from the main chain below the known checkpointed values, this protection only applies to nodes that have been able to download the honest chain to the checkpointed heights.

  We can protect all nodes from DoS from low-difficulty headers by adopting a different strategy: before we commit to storing a header in permanent storage, first verify that the header is part of a chain that has sufficiently high work (either `nMinimumChainWork`, or something comparable to our tip). This means that we will download headers from a given peer twice: once to verify the work on the chain, and a second time when permanently storing the headers.

  The p2p protocol doesn't provide an easy way for us to ensure that we receive the same headers during the second download of peer's headers chain. To ensure that a peer doesn't (say) give us the main chain in phase 1 to trick us into permanently storing an alternate, low-work chain in phase 2, we store commitments to the headers during our first download, which we validate in the second download.

  Some parameters must be chosen for commitment size/frequency in phase 1, and validation of commitments in phase 2. In this PR, those parameters are chosen to both (a) minimize the per-peer memory usage that an attacker could utilize, and (b) bound the expected amount of permanent memory that an attacker could get us to use to be well-below the memory growth that we'd get from the honest chain (where we expect 1 new block header every 10 minutes).

  After this PR, we should be able to remove checkpoints from our code, which is a nice philosophical change for us to make as well, as there has been confusion over the years about the role checkpoints play in Bitcoin's consensus algorithm.

  Thanks to Pieter Wuille for collaborating on this design.

ACKs for top commit:
  Sjors:
    re-tACK 3add234546
  mzumsande:
    re-ACK 3add234546
  sipa:
    re-ACK 3add234546
  glozow:
    ACK 3add234546

Tree-SHA512: e7789d65f62f72141b8899eb4a2fb3d0621278394d2d7adaa004675250118f89a4e4cb42777fe56649d744ec445ad95141e10f6def65f0a58b7b35b2e654a875

test/functional/feature_block.py

@@ -1297,7 +1297,7 @@ class FullBlockTest(BitcoinTestFramework):
         blocks2 = []
         for i in range(89, LARGE_REORG_SIZE + 89):
             blocks2.append(self.next_block("alt" + str(i)))
-        self.send_blocks(blocks2, False, force_send=True)
+        self.send_blocks(blocks2, False, force_send=False)
 
         # extend alt chain to trigger re-org
         block = self.next_block("alt" + str(chain1_tip + 1))

test/functional/p2p_compactblocks.py

@@ -615,6 +615,27 @@ class CompactBlocksTest(BitcoinTestFramework):
             bad_peer.send_message(msg)
         bad_peer.wait_for_disconnect()
 
+    def test_low_work_compactblocks(self, test_node):
+        # A compactblock with insufficient work won't get its header included
+        node = self.nodes[0]
+        hashPrevBlock = int(node.getblockhash(node.getblockcount() - 150), 16)
+        block = self.build_block_on_tip(node)
+        block.hashPrevBlock = hashPrevBlock
+        block.solve()
+
+        comp_block = HeaderAndShortIDs()
+        comp_block.initialize_from_block(block)
+        with self.nodes[0].assert_debug_log(['[net] Ignoring low-work compact block from peer 0']):
+            test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p()))
+
+        tips = node.getchaintips()
+        found = False
+        for x in tips:
+            if x["hash"] == block.hash:
+                found = True
+                break
+        assert not found
+
     def test_compactblocks_not_at_tip(self, test_node):
         node = self.nodes[0]
         # Test that requesting old compactblocks doesn't work.
@@ -833,6 +854,9 @@ class CompactBlocksTest(BitcoinTestFramework):
         self.log.info("Testing compactblock requests/announcements not at chain tip...")
         self.test_compactblocks_not_at_tip(self.segwit_node)
 
+        self.log.info("Testing handling of low-work compact blocks...")
+        self.test_low_work_compactblocks(self.segwit_node)
+
         self.log.info("Testing handling of incorrect blocktxn responses...")
         self.test_incorrect_blocktxn_response(self.segwit_node)
 

test/functional/p2p_dos_header_tree.py

@@ -22,6 +22,7 @@ class RejectLowDifficultyHeadersTest(BitcoinTestFramework):
         self.setup_clean_chain = True
         self.chain = 'testnet3'  # Use testnet chain because it has an early checkpoint
         self.num_nodes = 2
+        self.extra_args = [["-minimumchainwork=0x0"], ["-minimumchainwork=0x0"]]
 
     def add_options(self, parser):
         parser.add_argument(
@@ -62,7 +63,7 @@ class RejectLowDifficultyHeadersTest(BitcoinTestFramework):
 
         self.log.info("Feed all fork headers (succeeds without checkpoint)")
         # On node 0 it succeeds because checkpoints are disabled
-        self.restart_node(0, extra_args=['-nocheckpoints'])
+        self.restart_node(0, extra_args=['-nocheckpoints', "-minimumchainwork=0x0"])
         peer_no_checkpoint = self.nodes[0].add_p2p_connection(P2PInterface())
         peer_no_checkpoint.send_and_ping(msg_headers(self.headers_fork))
         assert {

test/functional/p2p_headers_sync_with_minchainwork.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019-2021 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test that we reject low difficulty headers to prevent our block tree from filling up with useless bloat"""
+
+from test_framework.test_framework import BitcoinTestFramework
+
+from test_framework.p2p import (
+    P2PInterface,
+)
+
+from test_framework.messages import (
+    msg_headers,
+)
+
+from test_framework.blocktools import (
+    NORMAL_GBT_REQUEST_PARAMS,
+    create_block,
+)
+
+from test_framework.util import assert_equal
+
+NODE1_BLOCKS_REQUIRED = 15
+NODE2_BLOCKS_REQUIRED = 2047
+
+
+class RejectLowDifficultyHeadersTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 3
+        # Node0 has no required chainwork; node1 requires 15 blocks on top of the genesis block; node2 requires 2047
+        self.extra_args = [["-minimumchainwork=0x0", "-checkblockindex=0"], ["-minimumchainwork=0x1f", "-checkblockindex=0"], ["-minimumchainwork=0x1000", "-checkblockindex=0"]]
+
+    def setup_network(self):
+        self.setup_nodes()
+        self.reconnect_all()
+        self.sync_all()
+
+    def disconnect_all(self):
+        self.disconnect_nodes(0, 1)
+        self.disconnect_nodes(0, 2)
+
+    def reconnect_all(self):
+        self.connect_nodes(0, 1)
+        self.connect_nodes(0, 2)
+
+    def test_chains_sync_when_long_enough(self):
+        self.log.info("Generate blocks on the node with no required chainwork, and verify nodes 1 and 2 have no new headers in their headers tree")
+        with self.nodes[1].assert_debug_log(expected_msgs=["[net] Ignoring low-work chain (height=14)"]), self.nodes[2].assert_debug_log(expected_msgs=["[net] Ignoring low-work chain (height=14)"]):
+            self.generate(self.nodes[0], NODE1_BLOCKS_REQUIRED-1, sync_fun=self.no_op)
+
+        for node in self.nodes[1:]:
+            chaintips = node.getchaintips()
+            assert(len(chaintips) == 1)
+            assert {
+                'height': 0,
+                'hash': '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206',
+                'branchlen': 0,
+                'status': 'active',
+            } in chaintips
+
+        self.log.info("Generate more blocks to satisfy node1's minchainwork requirement, and verify node2 still has no new headers in headers tree")
+        with self.nodes[2].assert_debug_log(expected_msgs=["[net] Ignoring low-work chain (height=15)"]):
+            self.generate(self.nodes[0], NODE1_BLOCKS_REQUIRED - self.nodes[0].getblockcount(), sync_fun=self.no_op)
+        self.sync_blocks(self.nodes[0:2])
+
+        assert {
+            'height': 0,
+            'hash': '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206',
+            'branchlen': 0,
+            'status': 'active',
+        } in self.nodes[2].getchaintips()
+
+        assert(len(self.nodes[2].getchaintips()) == 1)
+
+        self.log.info("Generate long chain for node0/node1")
+        self.generate(self.nodes[0], NODE2_BLOCKS_REQUIRED-self.nodes[0].getblockcount(), sync_fun=self.no_op)
+
+        self.log.info("Verify that node2 will sync the chain when it gets long enough")
+        self.sync_blocks()
+
+    def test_peerinfo_includes_headers_presync_height(self):
+        self.log.info("Test that getpeerinfo() includes headers presync height")
+
+        # Disconnect network, so that we can find our own peer connection more
+        # easily
+        self.disconnect_all()
+
+        p2p = self.nodes[0].add_p2p_connection(P2PInterface())
+        node = self.nodes[0]
+
+        # Ensure we have a long chain already
+        current_height = self.nodes[0].getblockcount()
+        if (current_height < 3000):
+            self.generate(node, 3000-current_height, sync_fun=self.no_op)
+
+        # Send a group of 2000 headers, forking from genesis.
+        new_blocks = []
+        hashPrevBlock = int(node.getblockhash(0), 16)
+        for i in range(2000):
+            block = create_block(hashprev = hashPrevBlock, tmpl=node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS))
+            block.solve()
+            new_blocks.append(block)
+            hashPrevBlock = block.sha256
+
+        headers_message = msg_headers(headers=new_blocks)
+        p2p.send_and_ping(headers_message)
+
+        # getpeerinfo should show a sync in progress
+        assert_equal(node.getpeerinfo()[0]['presynced_headers'], 2000)
+
+    def test_large_reorgs_can_succeed(self):
+        self.log.info("Test that a 2000+ block reorg, starting from a point that is more than 2000 blocks before a locator entry, can succeed")
+
+        self.sync_all() # Ensure all nodes are synced.
+        self.disconnect_all()
+
+        # locator(block at height T) will have heights:
+        # [T, T-1, ..., T-10, T-12, T-16, T-24, T-40, T-72, T-136, T-264,
+        #  T-520, T-1032, T-2056, T-4104, ...]
+        # So mine a number of blocks > 4104 to ensure that the first window of
+        # received headers during a sync are fully between locator entries.
+        BLOCKS_TO_MINE = 4110
+
+        self.generate(self.nodes[0], BLOCKS_TO_MINE, sync_fun=self.no_op)
+        self.generate(self.nodes[1], BLOCKS_TO_MINE+2, sync_fun=self.no_op)
+
+        self.reconnect_all()
+
+        self.sync_blocks(timeout=300) # Ensure tips eventually agree
+
+
+    def run_test(self):
+        self.test_chains_sync_when_long_enough()
+
+        self.test_large_reorgs_can_succeed()
+
+        self.test_peerinfo_includes_headers_presync_height()
+
+
+
+if __name__ == '__main__':
+    RejectLowDifficultyHeadersTest().main()

test/functional/p2p_unrequested_blocks.py

@@ -72,6 +72,13 @@ class AcceptBlockTest(BitcoinTestFramework):
     def setup_network(self):
         self.setup_nodes()
 
+    def check_hash_in_chaintips(self, node, blockhash):
+        tips = node.getchaintips()
+        for x in tips:
+            if x["hash"] == blockhash:
+                return True
+        return False
+
     def run_test(self):
         test_node = self.nodes[0].add_p2p_connection(P2PInterface())
         min_work_node = self.nodes[1].add_p2p_connection(P2PInterface())
@@ -89,10 +96,15 @@ class AcceptBlockTest(BitcoinTestFramework):
             blocks_h2[i].solve()
             block_time += 1
         test_node.send_and_ping(msg_block(blocks_h2[0]))
-        min_work_node.send_and_ping(msg_block(blocks_h2[1]))
+
+        with self.nodes[1].assert_debug_log(expected_msgs=[f"AcceptBlockHeader: not adding new block header {blocks_h2[1].hash}, missing anti-dos proof-of-work validation"]):
+            min_work_node.send_and_ping(msg_block(blocks_h2[1]))
 
         assert_equal(self.nodes[0].getblockcount(), 2)
         assert_equal(self.nodes[1].getblockcount(), 1)
+
+        # Ensure that the header of the second block was also not accepted by node1
+        assert_equal(self.check_hash_in_chaintips(self.nodes[1], blocks_h2[1].hash), False)
         self.log.info("First height 2 block accepted by node0; correctly rejected by node1")
 
         # 3. Send another block that builds on genesis.

test/functional/rpc_blockchain.py

@@ -452,8 +452,9 @@ class BlockchainTest(BitcoinTestFramework):
         # (Previously this was broken based on setting
         # `rpc/blockchain.cpp:latestblock` incorrectly.)
         #
-        b20hash = node.getblockhash(20)
-        b20 = node.getblock(b20hash)
+        fork_height = current_height - 100 # choose something vaguely near our tip
+        fork_hash = node.getblockhash(fork_height)
+        fork_block = node.getblock(fork_hash)
 
         def solve_and_send_block(prevhash, height, time):
             b = create_block(prevhash, create_coinbase(height), time)
@@ -461,10 +462,10 @@ class BlockchainTest(BitcoinTestFramework):
             peer.send_and_ping(msg_block(b))
             return b
 
-        b21f = solve_and_send_block(int(b20hash, 16), 21, b20['time'] + 1)
-        b22f = solve_and_send_block(b21f.sha256, 22, b21f.nTime + 1)
+        b1 = solve_and_send_block(int(fork_hash, 16), fork_height+1, fork_block['time'] + 1)
+        b2 = solve_and_send_block(b1.sha256, fork_height+2, b1.nTime + 1)
 
-        node.invalidateblock(b22f.hash)
+        node.invalidateblock(b2.hash)
 
         def assert_waitforheight(height, timeout=2):
             assert_equal(

test/functional/test_runner.py

@@ -186,6 +186,7 @@ BASE_SCRIPTS = [
     'wallet_signrawtransactionwithwallet.py --legacy-wallet',
     'wallet_signrawtransactionwithwallet.py --descriptors',
     'rpc_signrawtransactionwithkey.py',
+    'p2p_headers_sync_with_minchainwork.py',
     'rpc_rawtransaction.py --legacy-wallet',
     'wallet_groups.py --legacy-wallet',
     'wallet_transactiontime_rescan.py --descriptors',