Merge bitcoin/bitcoin#28793: contrib: Add asmap-tool

6abe772a17 contrib: Add asmap-tool (Fabian Jahr)

Pull request description:

  This adds `asmap.py` and `asmap-tool.py` from sipa's `nextgen` branch: https://github.com/sipa/asmap/tree/nextgen

  The motivation is that we should maintain the tooling for de- and encoding asmap files within the bitcoin core repository because it is not possible to use an asmap file that is not encoded.

  We already had an earlier version of `asmap.py` within the seeds contrib tools. The newer version only had a small amount of changes and is still compatible, so the old version is removed from contrib/seeds and the new version is made available to `makeseeds.py`.

ACKs for top commit:
  virtu:
    ACK [6abe772](6abe772a17)
  0xB10C:
    ACK 6abe772a17
  achow101:
    ACK 6abe772a17
  brunoerg:
    ACK 6abe772a17

Tree-SHA512: cc2a82ffa4eb46fa0ce4ca769dd82f8d0d2f37fc3652aa748eeb060e1142f9da4035008fe89433e2fd524a4dc153b7b9c085748944b49137b37009b0c0be8afb

contrib/asmap/README.md

@@ -0,0 +1,12 @@
+# ASMap Tool
+
+Tool for performing various operations on textual and binary asmap files,
+particularly encoding/compressing the raw data to the binary format that can
+be used in Bitcoin Core with the `-asmap` option.
+
+Example usage:
+```
+python3 asmap-tool.py encode /path/to/input.file /path/to/output.file
+python3 asmap-tool.py decode /path/to/input.file /path/to/output.file
+python3 asmap-tool.py diff /path/to/first.file /path/to/second.file
+```

contrib/asmap/asmap-tool.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 Pieter Wuille
+# Distributed under the MIT software license, see the accompanying
+# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
+
+import argparse
+import sys
+import ipaddress
+import math
+
+import asmap
+
+def load_file(input_file):
+    try:
+        contents = input_file.read()
+    except OSError as err:
+        sys.exit(f"Input file '{input_file.name}' cannot be read: {err.strerror}.")
+    try:
+        bin_asmap = asmap.ASMap.from_binary(contents)
+    except ValueError:
+        bin_asmap = None
+    txt_error = None
+    entries = None
+    try:
+        txt_contents = str(contents, encoding="utf-8")
+    except UnicodeError:
+        txt_error = "invalid UTF-8"
+        txt_contents = None
+    if txt_contents is not None:
+        entries = []
+        for line in txt_contents.split("\n"):
+            idx = line.find('#')
+            if idx >= 0:
+                line = line[:idx]
+            line = line.lstrip(' ').rstrip(' \t\r\n')
+            if len(line) == 0:
+                continue
+            fields = line.split(' ')
+            if len(fields) != 2:
+                txt_error = f"unparseable line '{line}'"
+                entries = None
+                break
+            prefix, asn = fields
+            if len(asn) <= 2 or asn[:2] != "AS" or any(c < '0' or c > '9' for c in asn[2:]):
+                txt_error = f"invalid ASN '{asn}'"
+                entries = None
+                break
+            try:
+                net = ipaddress.ip_network(prefix)
+            except ValueError:
+                txt_error = f"invalid network '{prefix}'"
+                entries = None
+                break
+            entries.append((asmap.net_to_prefix(net), int(asn[2:])))
+    if entries is not None and bin_asmap is not None and len(contents) > 0:
+        sys.exit(f"Input file '{input_file.name}' is ambiguous.")
+    if entries is not None:
+        state = asmap.ASMap()
+        state.update_multi(entries)
+        return state
+    if bin_asmap is not None:
+        return bin_asmap
+    sys.exit(f"Input file '{input_file.name}' is neither a valid binary asmap file nor valid text input ({txt_error}).")
+
+
+def save_binary(output_file, state, fill):
+    contents = state.to_binary(fill=fill)
+    try:
+        output_file.write(contents)
+        output_file.close()
+    except OSError as err:
+        sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
+
+def save_text(output_file, state, fill, overlapping):
+    for prefix, asn in state.to_entries(fill=fill, overlapping=overlapping):
+        net = asmap.prefix_to_net(prefix)
+        try:
+            print(f"{net} AS{asn}", file=output_file)
+        except OSError as err:
+            sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
+    try:
+        output_file.close()
+    except OSError as err:
+        sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
+
+def main():
+    parser = argparse.ArgumentParser(description="Tool for performing various operations on textual and binary asmap files.")
+    subparsers = parser.add_subparsers(title="valid subcommands", dest="subcommand")
+
+    parser_encode = subparsers.add_parser("encode", help="convert asmap data to binary format")
+    parser_encode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true",
+                               help="permit reassigning undefined network ranges arbitrarily to reduce size")
+    parser_encode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer,
+                               help="input asmap file (text or binary); default is stdin")
+    parser_encode.add_argument('outfile', nargs='?', type=argparse.FileType('wb'), default=sys.stdout.buffer,
+                               help="output binary asmap file; default is stdout")
+
+    parser_decode = subparsers.add_parser("decode", help="convert asmap data to text format")
+    parser_decode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true",
+                               help="permit reassigning undefined network ranges arbitrarily to reduce length")
+    parser_decode.add_argument('-n', '--nonoverlapping', dest="overlapping", default=True, action="store_false",
+                               help="output strictly non-overall ping network ranges (increases output size)")
+    parser_decode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer,
+                               help="input asmap file (text or binary); default is stdin")
+    parser_decode.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout,
+                               help="output text file; default is stdout")
+
+    parser_diff = subparsers.add_parser("diff", help="compute the difference between two asmap files")
+    parser_diff.add_argument('-i', '--ignore-unassigned', dest="ignore_unassigned", default=False, action="store_true",
+                             help="ignore unassigned ranges in the first input (useful when second input is filled)")
+    parser_diff.add_argument('infile1', type=argparse.FileType('rb'),
+                             help="first file to compare (text or binary)")
+    parser_diff.add_argument('infile2', type=argparse.FileType('rb'),
+                             help="second file to compare (text or binary)")
+
+    args = parser.parse_args()
+    if args.subcommand is None:
+        parser.print_help()
+    elif args.subcommand == "encode":
+        state = load_file(args.infile)
+        save_binary(args.outfile, state, fill=args.fill)
+    elif args.subcommand == "decode":
+        state = load_file(args.infile)
+        save_text(args.outfile, state, fill=args.fill, overlapping=args.overlapping)
+    elif args.subcommand == "diff":
+        state1 = load_file(args.infile1)
+        state2 = load_file(args.infile2)
+        ipv4_changed = 0
+        ipv6_changed = 0
+        for prefix, old_asn, new_asn in state1.diff(state2):
+            if args.ignore_unassigned and old_asn == 0:
+                continue
+            net = asmap.prefix_to_net(prefix)
+            if isinstance(net, ipaddress.IPv4Network):
+                ipv4_changed += 1 << (32 - net.prefixlen)
+            elif isinstance(net, ipaddress.IPv6Network):
+                ipv6_changed += 1 << (128 - net.prefixlen)
+            if new_asn == 0:
+                print(f"# {net} was AS{old_asn}")
+            elif old_asn == 0:
+                print(f"{net} AS{new_asn} # was unassigned")
+            else:
+                print(f"{net} AS{new_asn} # was AS{old_asn}")
+        ipv4_change_str = "" if ipv4_changed == 0 else f" (2^{math.log2(ipv4_changed):.2f})"
+        ipv6_change_str = "" if ipv6_changed == 0 else f" (2^{math.log2(ipv6_changed):.2f})"
+
+        print(
+            f"# {ipv4_changed}{ipv4_change_str} IPv4 addresses changed; "
+            f"{ipv6_changed}{ipv6_change_str} IPv6 addresses changed"
+        )
+    else:
+        parser.print_help()
+        sys.exit("No command provided.")
+
+if __name__ == '__main__':
+    main()

contrib/asmap/asmap.py

@@ -489,12 +489,14 @@ class ASMap:
                     if ctx not in ret or cand.size < ret[ctx].size:
                         ret[ctx] = cand
 
-            for ctx in set(left) | set(right):
+            union = set(left) | set(right)
+            sorted_union = sorted(union, key=lambda x: (x is None, x))
+            for ctx in sorted_union:
                 candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch)
                 candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch)
                 candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch)
             if not hole:
-                for ctx in set(ret) - set([None]):
+                for ctx in sorted(set(ret) - set([None])):
                     candidate(None, ctx, ret[ctx], _BinNode.make_default)
             if None in ret:
                 ret = {ctx:enc for ctx, enc in ret.items()

contrib/seeds/makeseeds.py

@@ -9,11 +9,14 @@
 import argparse
 import collections
 import ipaddress
+from pathlib import Path
 import re
 import sys
 from typing import Union
 
-from asmap import ASMap, net_to_prefix
+asmap_dir = Path(__file__).parent.parent / "asmap"
+sys.path.append(str(asmap_dir))
+from asmap import ASMap, net_to_prefix  # noqa: E402
 
 NSEEDS=512