Merge #19845: net: CNetAddr: add support to (un)serialize as ADDRv2

7be6ff6187 net: recognize TORv3/I2P/CJDNS networks (Vasil Dimov)
e0d73573a3 net: CNetAddr: add support to (un)serialize as ADDRv2 (Vasil Dimov)
fe42411b4b test: move HasReason so it can be reused (Vasil Dimov)
d2bb681f96 util: move HasPrefix() so it can be reused (Vasil Dimov)

Pull request description:

  (chopped off from #19031 to ease review)

  Add an optional support to serialize/unserialize `CNetAddr` in ADDRv2 format (BIP155). The new serialization is engaged by ORing a flag into the stream version.

  So far this is only used in tests to ensure the new code works as expected.

ACKs for top commit:
  Sjors:
    re-tACK 7be6ff6187
  sipa:
    re-utACK 7be6ff6187
  eriknylund:
    ACK 7be6ff6187 I built the PR on macOS Catalina 10.15.6, ran both tests and functional tests. I've reviewed the code and think the changes look good and according to BIP155. I verified that the added Base32 encoding test looks as proposed and working. I've run a node for a week only with Onion addresses `-onlynet=onion` without issues and I can connect to other peer reviewers running TorV3 on their nodes and I can connect both of my test nodes to each other.
  jonatack:
    re-ACK 7be6ff6187 per `git diff b9c46e0 7be6ff6`, debug build, ran/running bitcoind with this change and observed the log and `-netinfo` peer connections while connected as a tor v2 service to both tor v2 peers and also five tor v3 peers.
  hebasto:
    ACK 7be6ff6187, tested on Linux Mint 20 (x86_64): on top of this pull and #19031 I'm able to connect to onion v3 addresses, and jonatack is able to connect to my created onion v3 address.

Tree-SHA512: dc621411ac4393993aa3ccad10991717ec5f9f2643cae46a24a89802df0a33d6042994fc8ff2f0f397a3dbcd1c0e58fe4724305a2f9eb64d9342c3bdf784d9be

src/test/base32_tests.cpp

@@ -13,10 +13,13 @@ BOOST_AUTO_TEST_CASE(base32_testvectors)
 {
     static const std::string vstrIn[]  = {"","f","fo","foo","foob","fooba","foobar"};
     static const std::string vstrOut[] = {"","my======","mzxq====","mzxw6===","mzxw6yq=","mzxw6ytb","mzxw6ytboi======"};
+    static const std::string vstrOutNoPadding[] = {"","my","mzxq","mzxw6","mzxw6yq","mzxw6ytb","mzxw6ytboi"};
     for (unsigned int i=0; i<sizeof(vstrIn)/sizeof(vstrIn[0]); i++)
     {
         std::string strEnc = EncodeBase32(vstrIn[i]);
         BOOST_CHECK_EQUAL(strEnc, vstrOut[i]);
+        strEnc = EncodeBase32(vstrIn[i], false);
+        BOOST_CHECK_EQUAL(strEnc, vstrOutNoPadding[i]);
         std::string strDec = DecodeBase32(vstrOut[i]);
         BOOST_CHECK_EQUAL(strDec, vstrIn[i]);
     }

src/test/miner_tests.cpp

@@ -36,17 +36,6 @@ struct MinerTestingSetup : public TestingSetup {
 
 BOOST_FIXTURE_TEST_SUITE(miner_tests, MinerTestingSetup)
 
-// BOOST_CHECK_EXCEPTION predicates to check the specific validation error
-class HasReason {
-public:
-    explicit HasReason(const std::string& reason) : m_reason(reason) {}
-    bool operator() (const std::runtime_error& e) const {
-        return std::string(e.what()).find(m_reason) != std::string::npos;
-    };
-private:
-    const std::string m_reason;
-};
-
 static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE);
 
 BlockAssembler MinerTestingSetup::AssemblerForTest(const CChainParams& params)

src/test/net_tests.cpp

@@ -10,6 +10,7 @@
 #include <net.h>
 #include <netbase.h>
 #include <serialize.h>
+#include <span.h>
 #include <streams.h>
 #include <test/util/setup_common.h>
 #include <util/memory.h>
@@ -20,6 +21,7 @@
 
 #include <boost/test/unit_test.hpp>
 
+#include <ios>
 #include <memory>
 #include <string>
 
@@ -245,13 +247,38 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic)
     BOOST_CHECK_EQUAL(addr.ToString(), "1122:3344:5566:7788:9900:aabb:ccdd:eeff");
 
     // TORv2
-    addr.SetSpecial("6hzph5hv6337r6p2.onion");
+    BOOST_REQUIRE(addr.SetSpecial("6hzph5hv6337r6p2.onion"));
     BOOST_REQUIRE(addr.IsValid());
     BOOST_REQUIRE(addr.IsTor());
 
     BOOST_CHECK(!addr.IsBindAny());
     BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion");
 
+    // TORv3
+    const char* torv3_addr = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion";
+    BOOST_REQUIRE(addr.SetSpecial(torv3_addr));
+    BOOST_REQUIRE(addr.IsValid());
+    BOOST_REQUIRE(addr.IsTor());
+
+    BOOST_CHECK(!addr.IsBindAny());
+    BOOST_CHECK_EQUAL(addr.ToString(), torv3_addr);
+
+    // TORv3, broken, with wrong checksum
+    BOOST_CHECK(!addr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscsad.onion"));
+
+    // TORv3, broken, with wrong version
+    BOOST_CHECK(!addr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscrye.onion"));
+
+    // TORv3, malicious
+    BOOST_CHECK(!addr.SetSpecial(std::string{
+        "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd\0wtf.onion", 66}));
+
+    // TOR, bogus length
+    BOOST_CHECK(!addr.SetSpecial(std::string{"mfrggzak.onion"}));
+
+    // TOR, invalid base32
+    BOOST_CHECK(!addr.SetSpecial(std::string{"mf*g zak.onion"}));
+
     // Internal
     addr.SetInternal("esffpp");
     BOOST_REQUIRE(!addr.IsValid()); // "internal" is considered invalid
@@ -259,19 +286,286 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic)
 
     BOOST_CHECK(!addr.IsBindAny());
     BOOST_CHECK_EQUAL(addr.ToString(), "esffpvrt3wpeaygy.internal");
+
+    // Totally bogus
+    BOOST_CHECK(!addr.SetSpecial("totally bogus"));
 }
 
-BOOST_AUTO_TEST_CASE(cnetaddr_serialize)
+BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1)
 {
     CNetAddr addr;
     CDataStream s(SER_NETWORK, PROTOCOL_VERSION);
 
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000");
+    s.clear();
+
+    BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304");
+    s.clear();
+
+    BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b");
+    s.clear();
+
+    BOOST_REQUIRE(addr.SetSpecial("6hzph5hv6337r6p2.onion"));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "fd87d87eeb43f1f2f3f4f5f6f7f8f9fa");
+    s.clear();
+
+    BOOST_REQUIRE(addr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000");
+    s.clear();
+
     addr.SetInternal("a");
     s << addr;
     BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2");
     s.clear();
 }
 
+BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2)
+{
+    CNetAddr addr;
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION);
+    // Add ADDRV2_FORMAT to the version so that the CNetAddr
+    // serialize method produces an address in v2 format.
+    s.SetVersion(s.GetVersion() | ADDRV2_FORMAT);
+
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000");
+    s.clear();
+
+    BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "010401020304");
+    s.clear();
+
+    BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b");
+    s.clear();
+
+    BOOST_REQUIRE(addr.SetSpecial("6hzph5hv6337r6p2.onion"));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "030af1f2f3f4f5f6f7f8f9fa");
+    s.clear();
+
+    BOOST_REQUIRE(addr.SetSpecial("kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion"));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "042053cd5648488c4707914182655b7664034e09e66f7e8cbf1084e654eb56c5bd88");
+    s.clear();
+
+    BOOST_REQUIRE(addr.SetInternal("a"));
+    s << addr;
+    BOOST_CHECK_EQUAL(HexStr(s), "0210fd6b88c08724ca978112ca1bbdcafac2");
+    s.clear();
+}
+
+BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2)
+{
+    CNetAddr addr;
+    CDataStream s(SER_NETWORK, PROTOCOL_VERSION);
+    // Add ADDRV2_FORMAT to the version so that the CNetAddr
+    // unserialize method expects an address in v2 format.
+    s.SetVersion(s.GetVersion() | ADDRV2_FORMAT);
+
+    // Valid IPv4.
+    s << MakeSpan(ParseHex("01"          // network type (IPv4)
+                           "04"          // address length
+                           "01020304")); // address
+    s >> addr;
+    BOOST_CHECK(addr.IsValid());
+    BOOST_CHECK(addr.IsIPv4());
+    BOOST_CHECK_EQUAL(addr.ToString(), "1.2.3.4");
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid IPv4, valid length but address itself is shorter.
+    s << MakeSpan(ParseHex("01"      // network type (IPv4)
+                           "04"      // address length
+                           "0102")); // address
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("end of data"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Invalid IPv4, with bogus length.
+    s << MakeSpan(ParseHex("01"          // network type (IPv4)
+                           "05"          // address length
+                           "01020304")); // address
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("BIP155 IPv4 address with length 5 (should be 4)"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Invalid IPv4, with extreme length.
+    s << MakeSpan(ParseHex("01"          // network type (IPv4)
+                           "fd0102"      // address length (513 as CompactSize)
+                           "01020304")); // address
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("Address too long: 513 > 512"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Valid IPv6.
+    s << MakeSpan(ParseHex("02"                                  // network type (IPv6)
+                           "10"                                  // address length
+                           "0102030405060708090a0b0c0d0e0f10")); // address
+    s >> addr;
+    BOOST_CHECK(addr.IsValid());
+    BOOST_CHECK(addr.IsIPv6());
+    BOOST_CHECK_EQUAL(addr.ToString(), "102:304:506:708:90a:b0c:d0e:f10");
+    BOOST_REQUIRE(s.empty());
+
+    // Valid IPv6, contains embedded "internal".
+    s << MakeSpan(ParseHex(
+        "02"                                  // network type (IPv6)
+        "10"                                  // address length
+        "fd6b88c08724ca978112ca1bbdcafac2")); // address: 0xfd + sha256("bitcoin")[0:5] +
+                                              // sha256(name)[0:10]
+    s >> addr;
+    BOOST_CHECK(addr.IsInternal());
+    BOOST_CHECK_EQUAL(addr.ToString(), "zklycewkdo64v6wc.internal");
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid IPv6, with bogus length.
+    s << MakeSpan(ParseHex("02"    // network type (IPv6)
+                           "04"    // address length
+                           "00")); // address
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("BIP155 IPv6 address with length 4 (should be 16)"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Invalid IPv6, contains embedded IPv4.
+    s << MakeSpan(ParseHex("02"                                  // network type (IPv6)
+                           "10"                                  // address length
+                           "00000000000000000000ffff01020304")); // address
+    s >> addr;
+    BOOST_CHECK(!addr.IsValid());
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid IPv6, contains embedded TORv2.
+    s << MakeSpan(ParseHex("02"                                  // network type (IPv6)
+                           "10"                                  // address length
+                           "fd87d87eeb430102030405060708090a")); // address
+    s >> addr;
+    BOOST_CHECK(!addr.IsValid());
+    BOOST_REQUIRE(s.empty());
+
+    // Valid TORv2.
+    s << MakeSpan(ParseHex("03"                      // network type (TORv2)
+                           "0a"                      // address length
+                           "f1f2f3f4f5f6f7f8f9fa")); // address
+    s >> addr;
+    BOOST_CHECK(addr.IsValid());
+    BOOST_CHECK(addr.IsTor());
+    BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion");
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid TORv2, with bogus length.
+    s << MakeSpan(ParseHex("03"    // network type (TORv2)
+                           "07"    // address length
+                           "00")); // address
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("BIP155 TORv2 address with length 7 (should be 10)"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Valid TORv3.
+    s << MakeSpan(ParseHex("04"                               // network type (TORv3)
+                           "20"                               // address length
+                           "79bcc625184b05194975c28b66b66b04" // address
+                           "69f7f6556fb1ac3189a79b40dda32f1f"
+                           ));
+    s >> addr;
+    BOOST_CHECK(addr.IsValid());
+    BOOST_CHECK(addr.IsTor());
+    BOOST_CHECK_EQUAL(addr.ToString(),
+                      "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion");
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid TORv3, with bogus length.
+    s << MakeSpan(ParseHex("04" // network type (TORv3)
+                           "00" // address length
+                           "00" // address
+                           ));
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("BIP155 TORv3 address with length 0 (should be 32)"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Valid I2P.
+    s << MakeSpan(ParseHex("05"                               // network type (I2P)
+                           "20"                               // address length
+                           "a2894dabaec08c0051a481a6dac88b64" // address
+                           "f98232ae42d4b6fd2fa81952dfe36a87"));
+    s >> addr;
+    BOOST_CHECK(addr.IsValid());
+    BOOST_CHECK_EQUAL(addr.ToString(),
+                      "ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p");
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid I2P, with bogus length.
+    s << MakeSpan(ParseHex("05" // network type (I2P)
+                           "03" // address length
+                           "00" // address
+                           ));
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("BIP155 I2P address with length 3 (should be 32)"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Valid CJDNS.
+    s << MakeSpan(ParseHex("06"                               // network type (CJDNS)
+                           "10"                               // address length
+                           "fc000001000200030004000500060007" // address
+                           ));
+    s >> addr;
+    BOOST_CHECK(addr.IsValid());
+    BOOST_CHECK_EQUAL(addr.ToString(), "fc00:1:2:3:4:5:6:7");
+    BOOST_REQUIRE(s.empty());
+
+    // Invalid CJDNS, with bogus length.
+    s << MakeSpan(ParseHex("06" // network type (CJDNS)
+                           "01" // address length
+                           "00" // address
+                           ));
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("BIP155 CJDNS address with length 1 (should be 16)"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Unknown, with extreme length.
+    s << MakeSpan(ParseHex("aa"             // network type (unknown)
+                           "fe00000002"     // address length (CompactSize's MAX_SIZE)
+                           "01020304050607" // address
+                           ));
+    BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure,
+                          HasReason("Address too long: 33554432 > 512"));
+    BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input.
+    s.clear();
+
+    // Unknown, with reasonable length.
+    s << MakeSpan(ParseHex("aa"       // network type (unknown)
+                           "04"       // address length
+                           "01020304" // address
+                           ));
+    s >> addr;
+    BOOST_CHECK(!addr.IsValid());
+    BOOST_REQUIRE(s.empty());
+
+    // Unknown, with zero length.
+    s << MakeSpan(ParseHex("aa" // network type (unknown)
+                           "00" // address length
+                           ""   // address
+                           ));
+    s >> addr;
+    BOOST_CHECK(!addr.IsValid());
+    BOOST_REQUIRE(s.empty());
+}
+
 // prior to PR #14728, this test triggers an undefined behavior
 BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test)
 {

src/test/util/setup_common.h

@@ -153,4 +153,20 @@ CBlock getBlock13b8a();
 // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_*
 std::ostream& operator<<(std::ostream& os, const uint256& num);
 
+/**
+ * BOOST_CHECK_EXCEPTION predicates to check the specific validation error.
+ * Use as
+ * BOOST_CHECK_EXCEPTION(code that throws, exception type, HasReason("foo"));
+ */
+class HasReason {
+public:
+    explicit HasReason(const std::string& reason) : m_reason(reason) {}
+    template <typename E>
+    bool operator() (const E& e) const {
+        return std::string(e.what()).find(m_reason) != std::string::npos;
+    };
+private:
+    const std::string m_reason;
+};
+
 #endif