Merge #10387: Eventually connect to NODE_NETWORK_LIMITED peers

eb91835 Add setter for g_initial_block_download_completed (Jonas Schnelli)
3f56df5 [QA] add NODE_NETWORK_LIMITED address relay and sync test (Jonas Schnelli)
158e1a6 [QA] fix mininode CAddress ser/deser (Jonas Schnelli)
fa999af [QA] Allow addrman loopback tests (add debug option -addrmantest) (Jonas Schnelli)
6fe57bd Connect to peers signaling NODE_NETWORK_LIMITED when out-of-IBD (Jonas Schnelli)
31c45a9 Accept addresses with NODE_NETWORK_LIMITED flag (Jonas Schnelli)

Pull request description:

  Eventually connect to peers signalling NODE_NETWORK_LIMITED if we are out of IBD.
  Accept and relay NODE_NETWORK_LIMITED peers in addrman.

Tree-SHA512: 8a238fc97f767f81cae1866d6cc061390f23a72af4a711d2f7158c77f876017986abb371d213d1c84019eef7be4ca951e8e6f83fda36769c4e1a1d763f787037

test/functional/p2p_node_network_limited.py

@@ -8,16 +8,21 @@ Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correc
 and that it responds to getdata requests for blocks correctly:
     - send a block within 288 + 2 of the tip
     - disconnect peers who request blocks older than that."""
-from test_framework.messages import CInv, msg_getdata
-from test_framework.mininode import NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS, NetworkThread, P2PInterface
+from test_framework.messages import CInv, msg_getdata, msg_verack
+from test_framework.mininode import NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS, P2PInterface, wait_until, mininode_lock, network_thread_start, network_thread_join
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal
+from test_framework.util import assert_equal, disconnect_nodes, connect_nodes_bi, sync_blocks
 
 class P2PIgnoreInv(P2PInterface):
+    firstAddrnServices = 0
     def on_inv(self, message):
         # The node will send us invs for other blocks. Ignore them.
         pass
-
+    def on_addr(self, message):
+        self.firstAddrnServices = message.addrs[0].nServices
+    def wait_for_addr(self, timeout=5):
+        test_function = lambda: self.last_message.get("addr")
+        wait_until(test_function, timeout=timeout, lock=mininode_lock)
     def send_getdata_for_block(self, blockhash):
         getdata_request = msg_getdata()
         getdata_request.inv.append(CInv(2, int(blockhash, 16)))
@@ -26,12 +31,24 @@ class P2PIgnoreInv(P2PInterface):
 class NodeNetworkLimitedTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
-        self.num_nodes = 1
-        self.extra_args = [['-prune=550']]
+        self.num_nodes = 3
+        self.extra_args = [['-prune=550', '-addrmantest'], [], []]
+
+    def disconnect_all(self):
+        disconnect_nodes(self.nodes[0], 1)
+        disconnect_nodes(self.nodes[1], 0)
+        disconnect_nodes(self.nodes[2], 1)
+        disconnect_nodes(self.nodes[2], 0)
+        disconnect_nodes(self.nodes[0], 2)
+        disconnect_nodes(self.nodes[1], 2)
+
+    def setup_network(self):
+        super(NodeNetworkLimitedTest, self).setup_network()
+        self.disconnect_all()
 
     def run_test(self):
         node = self.nodes[0].add_p2p_connection(P2PIgnoreInv())
-        NetworkThread().start()
+        network_thread_start()
         node.wait_for_verack()
 
         expected_services = NODE_BLOOM | NODE_WITNESS | NODE_NETWORK_LIMITED
@@ -43,7 +60,9 @@ class NodeNetworkLimitedTest(BitcoinTestFramework):
         assert_equal(int(self.nodes[0].getnetworkinfo()['localservices'], 16), expected_services)
 
         self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.")
-        blocks = self.nodes[0].generate(292)
+        connect_nodes_bi(self.nodes, 0, 1)
+        blocks = self.nodes[1].generate(292)
+        sync_blocks([self.nodes[0], self.nodes[1]])
 
         self.log.info("Make sure we can max retrive block at tip-288.")
         node.send_getdata_for_block(blocks[1])  # last block in valid range
@@ -53,5 +72,48 @@ class NodeNetworkLimitedTest(BitcoinTestFramework):
         node.send_getdata_for_block(blocks[0])  # first block outside of the 288+2 limit
         node.wait_for_disconnect(5)
 
+        self.log.info("Check local address relay, do a fresh connection.")
+        self.nodes[0].disconnect_p2ps()
+        network_thread_join()
+        node1 = self.nodes[0].add_p2p_connection(P2PIgnoreInv())
+        network_thread_start()
+        node1.wait_for_verack()
+        node1.send_message(msg_verack())
+
+        node1.wait_for_addr()
+        #must relay address with NODE_NETWORK_LIMITED
+        assert_equal(node1.firstAddrnServices, 1036)
+
+        self.nodes[0].disconnect_p2ps()
+        node1.wait_for_disconnect()
+
+        # connect unsynced node 2 with pruned NODE_NETWORK_LIMITED peer
+        # because node 2 is in IBD and node 0 is a NODE_NETWORK_LIMITED peer, sync must not be possible
+        connect_nodes_bi(self.nodes, 0, 2)
+        try:
+            sync_blocks([self.nodes[0], self.nodes[2]], timeout=5)
+        except:
+            pass
+        # node2 must remain at heigh 0
+        assert_equal(self.nodes[2].getblockheader(self.nodes[2].getbestblockhash())['height'], 0)
+
+        # now connect also to node 1 (non pruned)
+        connect_nodes_bi(self.nodes, 1, 2)
+
+        # sync must be possible
+        sync_blocks(self.nodes)
+
+        # disconnect all peers
+        self.disconnect_all()
+
+        # mine 10 blocks on node 0 (pruned node)
+        self.nodes[0].generate(10)
+
+        # connect node1 (non pruned) with node0 (pruned) and check if the can sync
+        connect_nodes_bi(self.nodes, 0, 1)
+
+        # sync must be possible, node 1 is no longer in IBD and should therefore connect to node 0 (NODE_NETWORK_LIMITED)
+        sync_blocks([self.nodes[0], self.nodes[1]])
+
 if __name__ == '__main__':
     NodeNetworkLimitedTest().main()

test/functional/test_framework/messages.py

@@ -186,19 +186,24 @@ def ToHex(obj):
 
 class CAddress():
     def __init__(self):
+        self.time = 0
         self.nServices = 1
         self.pchReserved = b"\x00" * 10 + b"\xff" * 2
         self.ip = "0.0.0.0"
         self.port = 0
 
-    def deserialize(self, f):
+    def deserialize(self, f, with_time=True):
+        if with_time:
+            self.time = struct.unpack("<i", f.read(4))[0]
         self.nServices = struct.unpack("<Q", f.read(8))[0]
         self.pchReserved = f.read(12)
         self.ip = socket.inet_ntoa(f.read(4))
         self.port = struct.unpack(">H", f.read(2))[0]
 
-    def serialize(self):
+    def serialize(self, with_time=True):
         r = b""
+        if with_time:
+            r += struct.pack("<i", self.time)
         r += struct.pack("<Q", self.nServices)
         r += self.pchReserved
         r += socket.inet_aton(self.ip)
@@ -856,11 +861,11 @@ class msg_version():
         self.nServices = struct.unpack("<Q", f.read(8))[0]
         self.nTime = struct.unpack("<q", f.read(8))[0]
         self.addrTo = CAddress()
-        self.addrTo.deserialize(f)
+        self.addrTo.deserialize(f, False)
 
         if self.nVersion >= 106:
             self.addrFrom = CAddress()
-            self.addrFrom.deserialize(f)
+            self.addrFrom.deserialize(f, False)
             self.nNonce = struct.unpack("<Q", f.read(8))[0]
             self.strSubVer = deser_string(f)
         else:
@@ -888,8 +893,8 @@ class msg_version():
         r += struct.pack("<i", self.nVersion)
         r += struct.pack("<Q", self.nServices)
         r += struct.pack("<q", self.nTime)
-        r += self.addrTo.serialize()
-        r += self.addrFrom.serialize()
+        r += self.addrTo.serialize(False)
+        r += self.addrFrom.serialize(False)
         r += struct.pack("<Q", self.nNonce)
         r += ser_string(self.strSubVer)
         r += struct.pack("<i", self.nStartingHeight)