Merge #18112: Serialization improvements step 5 (blockencodings)

353f376277 Convert blockencodings.h to new serialization framework (Pieter Wuille)
e574fff53e Add CustomUintFormatter (Pieter Wuille)
10633398f2 Add DifferenceFormatter (Russell Yanofsky)
56dd9f04c7 Make VectorFormatter support stateful formatters (Russell Yanofsky)
3ca574cef0 Convert CCompactSize to proper formatter (Pieter Wuille)

Pull request description:

  This is probably the most involved change in the sequence of changes extracted from #10785.

  In order to implement the differential encoding of BIP152, this change changes `VectorFormatter` to permit a stateful sub-formatter, which is then used by `DifferenceFormatter`. A `CustomUintFormatter` is added as well to do the 48-bit serialization of short ids.

ACKs for top commit:
  laanwj:
    ACK 353f376277, nice change
  ryanofsky:
    Code review ACK 353f376277. Only changes since last review are suggested assert change and MASK->MAX rename

Tree-SHA512: 976618991a8be62ba0738725b7cfa166a56cde998ebf1031ba6f28557032f1577b666ac7ae25cd498c0e1e740108c3c56a342620b724df41d6cc9d8bdafac037

src/serialize.h

@@ -500,7 +500,7 @@ static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&
 
 #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
 #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
-#define COMPACTSIZE(obj) CCompactSize(REF(obj))
+#define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
 #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
 
 /** Serialization wrapper class for integers in VarInt format. */
@@ -518,6 +518,28 @@ struct VarIntFormatter
     }
 };
 
+template<int Bytes>
+struct CustomUintFormatter
+{
+    static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
+    static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
+
+    template <typename Stream, typename I> void Ser(Stream& s, I v)
+    {
+        if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
+        uint64_t raw = htole64(v);
+        s.write((const char*)&raw, Bytes);
+    }
+
+    template <typename Stream, typename I> void Unser(Stream& s, I& v)
+    {
+        static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small");
+        uint64_t raw = 0;
+        s.read((char*)&raw, Bytes);
+        v = le64toh(raw);
+    }
+};
+
 /** Serialization wrapper class for big-endian integers.
  *
  * Use this wrapper around integer types that are stored in memory in native
@@ -552,21 +574,26 @@ public:
     }
 };
 
-class CCompactSize
+/** Formatter for integers in CompactSize format. */
+struct CompactSizeFormatter
 {
-protected:
-    uint64_t &n;
-public:
-    explicit CCompactSize(uint64_t& nIn) : n(nIn) { }
-
-    template<typename Stream>
-    void Serialize(Stream &s) const {
-        WriteCompactSize<Stream>(s, n);
+    template<typename Stream, typename I>
+    void Unser(Stream& s, I& v)
+    {
+        uint64_t n = ReadCompactSize<Stream>(s);
+        if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
+            throw std::ios_base::failure("CompactSize exceeds limit of type");
+        }
+        v = n;
     }
 
-    template<typename Stream>
-    void Unserialize(Stream& s) {
-        n = ReadCompactSize<Stream>(s);
+    template<typename Stream, typename I>
+    void Ser(Stream& s, I v)
+    {
+        static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
+        static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
+
+        WriteCompactSize<Stream>(s, v);
     }
 };
 
@@ -613,7 +640,7 @@ BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
  * as a vector of VarInt-encoded integers.
  *
  * V is not required to be an std::vector type. It works for any class that
- * exposes a value_type, size, reserve, push_back, and const iterators.
+ * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
  */
 template<class Formatter>
 struct VectorFormatter
@@ -621,15 +648,17 @@ struct VectorFormatter
     template<typename Stream, typename V>
     void Ser(Stream& s, const V& v)
     {
+        Formatter formatter;
         WriteCompactSize(s, v.size());
         for (const typename V::value_type& elem : v) {
-            s << Using<Formatter>(elem);
+            formatter.Ser(s, elem);
         }
     }
 
     template<typename Stream, typename V>
     void Unser(Stream& s, V& v)
     {
+        Formatter formatter;
         v.clear();
         size_t size = ReadCompactSize(s);
         size_t allocated = 0;
@@ -641,9 +670,8 @@ struct VectorFormatter
             allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
             v.reserve(allocated);
             while (v.size() < allocated) {
-                typename V::value_type val;
-                s >> Using<Formatter>(val);
-                v.push_back(std::move(val));
+                v.emplace_back();
+                formatter.Unser(s, v.back());
             }
         }
     };