From da384a286bd84a97e7ebe7a64654c5be20ab2df1 Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Wed, 20 Sep 2023 19:37:45 +0200 Subject: [PATCH 1/2] rpc: getrawaddrman for addrman entries Exposing address manager table entries in a hidden RPC allows to introspect addrman tables in tests and during development. As response JSON object the following FORMAT1 is choosen: { "table": { "/": { "address": "..", "port": .., ... }, "/": { "address": "..", "port": .., ... }, "/": { "address": "..", "port": .., ... }, ... } } An alternative would be FORMAT2 { "table": { "bucket": { "position": { "address": "..", "port": .., ... }, "position": { "address": "..", "port": .., ... }, .. }, "bucket": { "position": { "address": "..", "port": .., ... }, .. }, } } FORMAT1 and FORMAT2 have different encodings for the location of the address in the address manager. While FORMAT2 might be easier to process for downstream tools, it also mimics internal addrman mappings, which might change at some point. Users not interested in the address location can ignore the location key. They don't have to adapt to a new RPC response format, when the internal addrman layout changes. Additionally, FORMAT1 is also slightly easier to to iterate in downstream tools. The RPC response-building implemenation complexcity is lower with FORMAT1 as we can more easily build a "/" key than a multiple "bucket" objects with multiple "position" objects (FORMAT2). --- src/addrman.cpp | 38 +++++++++++++++++++++++ src/addrman.h | 14 ++++++++- src/addrman_impl.h | 5 ++++ src/rpc/net.cpp | 70 +++++++++++++++++++++++++++++++++++++++++++ src/test/fuzz/rpc.cpp | 1 + 5 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/addrman.cpp b/src/addrman.cpp index 212baab9d45..6ce9c81c63a 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -838,6 +838,30 @@ std::vector AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct return addresses; } +std::vector> AddrManImpl::GetEntries_(bool from_tried) const +{ + AssertLockHeld(cs); + + const int bucket_count = from_tried ? ADDRMAN_TRIED_BUCKET_COUNT : ADDRMAN_NEW_BUCKET_COUNT; + std::vector> infos; + for (int bucket = 0; bucket < bucket_count; ++bucket) { + for (int position = 0; position < ADDRMAN_BUCKET_SIZE; ++position) { + int id = GetEntry(from_tried, bucket, position); + if (id >= 0) { + AddrInfo info = mapInfo.at(id); + AddressPosition location = AddressPosition( + from_tried, + /*multiplicity_in=*/from_tried ? 1 : info.nRefCount, + bucket, + position); + infos.push_back(std::make_pair(info, location)); + } + } + } + + return infos; +} + void AddrManImpl::Connected_(const CService& addr, NodeSeconds time) { AssertLockHeld(cs); @@ -1199,6 +1223,15 @@ std::vector AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, return addresses; } +std::vector> AddrManImpl::GetEntries(bool from_tried) const +{ + LOCK(cs); + Check(); + auto addrInfos = GetEntries_(from_tried); + Check(); + return addrInfos; +} + void AddrManImpl::Connected(const CService& addr, NodeSeconds time) { LOCK(cs); @@ -1289,6 +1322,11 @@ std::vector AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std return m_impl->GetAddr(max_addresses, max_pct, network); } +std::vector> AddrMan::GetEntries(bool use_tried) const +{ + return m_impl->GetEntries(use_tried); +} + void AddrMan::Connected(const CService& addr, NodeSeconds time) { m_impl->Connected(addr, time); diff --git a/src/addrman.h b/src/addrman.h index f41687dcffc..4d44c943ac8 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -25,11 +25,12 @@ public: }; class AddrManImpl; +class AddrInfo; /** Default for -checkaddrman */ static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; -/** Test-only struct, capturing info about an address in AddrMan */ +/** Location information for an address in AddrMan */ struct AddressPosition { // Whether the address is in the new or tried table const bool tried; @@ -168,6 +169,17 @@ public: */ std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const; + /** + * Returns an information-location pair for all addresses in the selected addrman table. + * If an address appears multiple times in the new table, an information-location pair + * is returned for each occurence. Addresses only ever appear once in the tried table. + * + * @param[in] from_tried Selects which table to return entries from. + * + * @return A vector consisting of pairs of AddrInfo and AddressPosition. + */ + std::vector> GetEntries(bool from_tried) const; + /** We have successfully connected to this peer. Calling this function * updates the CAddress's nTime, which is used in our IsTerrible() * decisions and gossiped to peers. Callers should be careful that updating diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 1cfaca04a3d..512f085a21f 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -132,6 +132,9 @@ public: std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + std::vector> GetEntries(bool from_tried) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + void Connected(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); @@ -260,6 +263,8 @@ private: std::vector GetAddr_(size_t max_addresses, size_t max_pct, std::optional network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector> GetEntries_(bool from_tried) const EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected_(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 6af62641bda..8ebc6a73cf7 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1063,6 +1064,74 @@ static RPCHelpMan getaddrmaninfo() }; } +UniValue AddrmanEntryToJSON(const AddrInfo& info) +{ + UniValue ret(UniValue::VOBJ); + ret.pushKV("address", info.ToStringAddr()); + ret.pushKV("port", info.GetPort()); + ret.pushKV("services", (uint64_t)info.nServices); + ret.pushKV("time", int64_t{TicksSinceEpoch(info.nTime)}); + ret.pushKV("network", GetNetworkName(info.GetNetClass())); + ret.pushKV("source", info.source.ToStringAddr()); + ret.pushKV("source_network", GetNetworkName(info.source.GetNetClass())); + return ret; +} + +UniValue AddrmanTableToJSON(const std::vector>& tableInfos) +{ + UniValue table(UniValue::VOBJ); + for (const auto& e : tableInfos) { + AddrInfo info = e.first; + AddressPosition location = e.second; + std::ostringstream key; + key << location.bucket << "/" << location.position; + // Address manager tables have unique entries so there is no advantage + // in using UniValue::pushKV, which checks if the key already exists + // in O(N). UniValue::pushKVEnd is used instead which currently is O(1). + table.pushKVEnd(key.str(), AddrmanEntryToJSON(info)); + } + return table; +} + +static RPCHelpMan getrawaddrman() +{ + return RPCHelpMan{"getrawaddrman", + "EXPERIMENTAL warning: this call may be changed in future releases.\n" + "\nReturns information on all address manager entries for the new and tried tables.\n", + {}, + RPCResult{ + RPCResult::Type::OBJ_DYN, "", "", { + {RPCResult::Type::OBJ_DYN, "table", "buckets with addresses in the address manager table ( new, tried )", { + {RPCResult::Type::OBJ, "bucket/position", "the location in the address manager table (/)", { + {RPCResult::Type::STR, "address", "The address of the node"}, + {RPCResult::Type::NUM, "port", "The port number of the node"}, + {RPCResult::Type::STR, "network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the address"}, + {RPCResult::Type::NUM, "services", "The services offered by the node"}, + {RPCResult::Type::NUM_TIME, "time", "The " + UNIX_EPOCH_TIME + " when the node was last seen"}, + {RPCResult::Type::STR, "source", "The address that relayed the address to us"}, + {RPCResult::Type::STR, "source_network", "The network (" + Join(GetNetworkNames(), ", ") + ") of the source address"}, + }} + }} + } + }, + RPCExamples{ + HelpExampleCli("getrawaddrman", "") + + HelpExampleRpc("getrawaddrman", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + NodeContext& node = EnsureAnyNodeContext(request.context); + if (!node.addrman) { + throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Address manager functionality missing or disabled"); + } + + UniValue ret(UniValue::VOBJ); + ret.pushKV("new", AddrmanTableToJSON(node.addrman->GetEntries(false))); + ret.pushKV("tried", AddrmanTableToJSON(node.addrman->GetEntries(true))); + return ret; + }, + }; +} + void RegisterNetRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -1083,6 +1152,7 @@ void RegisterNetRPCCommands(CRPCTable& t) {"hidden", &addpeeraddress}, {"hidden", &sendmsgtopeer}, {"hidden", &getaddrmaninfo}, + {"hidden", &getrawaddrman}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 7e9a18e1d0b..cffda751df2 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -140,6 +140,7 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnodeaddresses", "getpeerinfo", "getprioritisedtransactions", + "getrawaddrman", "getrawmempool", "getrawtransaction", "getrpcinfo", From 352d5eb2a9e89cff4a2815d94a9d81fcc20c4b2c Mon Sep 17 00:00:00 2001 From: 0xb10c Date: Sat, 23 Sep 2023 16:54:37 +0200 Subject: [PATCH 2/2] test: getrawaddrman RPC Test that the getrawaddrman returns the addresses in the new and tried tables. We can't check the buckets and positions as these are not deterministic (yet). --- test/functional/rpc_net.py | 112 +++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 117802b812c..da68066fc5f 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -12,6 +12,7 @@ from itertools import product import time import test_framework.messages +from test_framework.netutil import ADDRMAN_NEW_BUCKET_COUNT, ADDRMAN_TRIED_BUCKET_COUNT, ADDRMAN_BUCKET_SIZE from test_framework.p2p import ( P2PInterface, P2P_SERVICES, @@ -67,6 +68,7 @@ class NetTest(BitcoinTestFramework): self.test_addpeeraddress() self.test_sendmsgtopeer() self.test_getaddrmaninfo() + self.test_getrawaddrman() def test_connection_count(self): self.log.info("Test getconnectioncount") @@ -386,5 +388,115 @@ class NetTest(BitcoinTestFramework): assert_equal(res[net]["tried"], 0) assert_equal(res[net]["total"], 0) + def test_getrawaddrman(self): + self.log.info("Test getrawaddrman") + node = self.nodes[1] + + self.log.debug("Test that getrawaddrman is a hidden RPC") + # It is hidden from general help, but its detailed help may be called directly. + assert "getrawaddrman" not in node.help() + assert "getrawaddrman" in node.help("getrawaddrman") + + def check_addr_information(result, expected): + """Utility to compare a getrawaddrman result entry with an expected entry""" + assert_equal(result["address"], expected["address"]) + assert_equal(result["port"], expected["port"]) + assert_equal(result["services"], expected["services"]) + assert_equal(result["network"], expected["network"]) + assert_equal(result["source"], expected["source"]) + assert_equal(result["source_network"], expected["source_network"]) + # To avoid failing on slow test runners, use a 10s vspan here. + assert_approx(result["time"], time.time(), vspan=10) + + def check_getrawaddrman_entries(expected): + """Utility to compare a getrawaddrman result with expected addrman contents""" + getrawaddrman = node.getrawaddrman() + getaddrmaninfo = node.getaddrmaninfo() + for (table_name, table_info) in expected.items(): + assert_equal(len(getrawaddrman[table_name]), len(table_info["entries"])) + assert_equal(len(getrawaddrman[table_name]), getaddrmaninfo["all_networks"][table_name]) + + for bucket_position in getrawaddrman[table_name].keys(): + bucket = int(bucket_position.split("/")[0]) + position = int(bucket_position.split("/")[1]) + + # bucket and position only be sanity checked here as the + # test-addrman isn't deterministic + assert 0 <= int(bucket) < table_info["bucket_count"] + assert 0 <= int(position) < ADDRMAN_BUCKET_SIZE + + entry = getrawaddrman[table_name][bucket_position] + expected_entry = list(filter(lambda e: e["address"] == entry["address"], table_info["entries"]))[0] + check_addr_information(entry, expected_entry) + + # we expect one addrman new and tried table entry, which were added in a previous test + expected = { + "new": { + "bucket_count": ADDRMAN_NEW_BUCKET_COUNT, + "entries": [ + { + "address": "2.0.0.0", + "port": 8333, + "services": 9, + "network": "ipv4", + "source": "2.0.0.0", + "source_network": "ipv4", + } + ] + }, + "tried": { + "bucket_count": ADDRMAN_TRIED_BUCKET_COUNT, + "entries": [ + { + "address": "1.2.3.4", + "port": 8333, + "services": 9, + "network": "ipv4", + "source": "1.2.3.4", + "source_network": "ipv4", + } + ] + } + } + + self.log.debug("Test that the getrawaddrman contains information about the addresses added in a previous test") + check_getrawaddrman_entries(expected) + + self.log.debug("Add one new address to each addrman table") + expected["new"]["entries"].append({ + "address": "2803:0:1234:abcd::1", + "services": 9, + "network": "ipv6", + "source": "2803:0:1234:abcd::1", + "source_network": "ipv6", + "port": -1, # set once addpeeraddress is successful + }) + expected["tried"]["entries"].append({ + "address": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", + "services": 9, + "network": "onion", + "source": "nrfj6inpyf73gpkyool35hcmne5zwfmse3jl3aw23vk7chdemalyaqad.onion", + "source_network": "onion", + "port": -1, # set once addpeeraddress is successful + }) + + port = 0 + for (table_name, table_info) in expected.items(): + # There's a slight chance that the to-be-added address collides with an already + # present table entry. To avoid this, we increment the port until an address has been + # added. Incrementing the port changes the position in the new table bucket (bucket + # stays the same) and changes both the bucket and the position in the tried table. + while True: + if node.addpeeraddress(address=table_info["entries"][1]["address"], port=port, tried=table_name == "tried")["success"]: + table_info["entries"][1]["port"] = port + self.log.debug(f"Added {table_info['entries'][1]['address']} to {table_name} table") + break + else: + port += 1 + + self.log.debug("Test that the newly added addresses appear in getrawaddrman") + check_getrawaddrman_entries(expected) + + if __name__ == '__main__': NetTest().main()