negentropy: fuzz testing, move accumulator to vector package.

nip77/negentropy/encoding.go

@@ -117,10 +117,10 @@ func writeVarInt(w *StringHexWriter, n int) {
 		return
 	}
 
-	w.WriteBytes(encodeVarInt(n))
+	w.WriteBytes(EncodeVarInt(n))
 }
 
-func encodeVarInt(n int) []byte {
+func EncodeVarInt(n int) []byte {
 	if n == 0 {
 		return []byte{0}
 	}

nip77/negentropy/fuzz_test.go

@@ -0,0 +1,115 @@
+package negentropy_test
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"math/rand/v2"
+	"slices"
+	"sync"
+	"testing"
+
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy"
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy/storage/vector"
+	"github.com/stretchr/testify/require"
+)
+
+func FuzzWhatever(f *testing.F) {
+	var sectors uint = 1
+	var sectorSizeAvg uint = 10
+	var pctChance uint = 5
+	var frameSizeLimit uint = 0
+	f.Add(sectors, sectorSizeAvg, pctChance, frameSizeLimit)
+	f.Fuzz(func(t *testing.T, sectors uint, sectorSizeAvg uint, pctChance uint, frameSizeLimit uint) {
+		rand := rand.New(rand.NewPCG(1, 1000))
+		sectorSizeAvg += 1 // prevent divide by zero
+		frameSizeLimit += 4096
+		pctChance = pctChance % 100
+
+		// prepare the two sides
+		s1 := vector.New()
+		l1 := make([]string, 0, 500)
+		neg1 := negentropy.New(s1, int(frameSizeLimit))
+		s2 := vector.New()
+		l2 := make([]string, 0, 500)
+		neg2 := negentropy.New(s2, int(frameSizeLimit))
+
+		start := 0
+		for s := 0; s < int(sectors); s++ {
+			diff := rand.Uint() % sectorSizeAvg
+			if rand.IntN(2) == 0 {
+				diff = -diff
+			}
+			sectorSize := sectorSizeAvg + diff
+
+			for i := 0; i < int(sectorSize); i++ {
+				item := start + i
+
+				rnd := sha256.Sum256(binary.BigEndian.AppendUint64(nil, uint64(item)))
+				id := fmt.Sprintf("%x%056d", rnd[0:4], item)
+
+				if rand.IntN(100) < int(pctChance) {
+					s1.Insert(nostr.Timestamp(item), id)
+					l1 = append(l1, id)
+				}
+				if rand.IntN(100) < int(pctChance) {
+					id := fmt.Sprintf("%064d", item)
+					s2.Insert(nostr.Timestamp(item), id)
+					l2 = append(l2, id)
+				}
+			}
+
+			start += int(sectorSize)
+		}
+
+		// fmt.Println(neg1.Name(), "initial", l1)
+		// fmt.Println(neg2.Name(), "initial", l2)
+
+		wg := sync.WaitGroup{}
+		wg.Add(2)
+		go func() {
+			for item := range neg1.Haves {
+				// fmt.Println("have", item)
+				l2 = append(l2, item)
+			}
+			wg.Done()
+		}()
+		go func() {
+			for item := range neg1.HaveNots {
+				// fmt.Println("havenot", item)
+				l1 = append(l1, item)
+			}
+			wg.Done()
+		}()
+
+		msg := neg1.Start()
+		next := neg2
+
+		for {
+			var err error
+			// fmt.Println(next.Name(), "handling", msg)
+			msg, err = next.Reconcile(msg)
+			if err != nil {
+				panic(err)
+			}
+
+			if msg == "" {
+				break
+			}
+
+			if next == neg1 {
+				next = neg2
+			} else {
+				next = neg1
+			}
+		}
+
+		wg.Wait()
+		slices.Sort(l1)
+		l1 = slices.Compact(l1)
+		slices.Sort(l2)
+		l2 = slices.Compact(l2)
+		require.ElementsMatch(t, l1, l2)
+	})
+}

nip77/negentropy/hex.go

@@ -36,9 +36,6 @@ func (r *StringHexReader) ReadHexByte() (byte, error) {
 }
 
 func (r *StringHexReader) ReadString(size int) (string, error) {
-	if size == 0 {
-		return "", nil
-	}
 	r.idx += size
 	if len(r.source) < r.idx {
 		return "", io.EOF

nip77/negentropy/negentropy.go

@@ -134,8 +134,8 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 			skipping = true
 
 		case FingerprintMode:
-			var theirFingerprint [FingerprintSize]byte
+			theirFingerprint, err := reader.ReadString(FingerprintSize * 2)
-			if err := reader.ReadHexBytes(theirFingerprint[:]); err != nil {
+			if err != nil {
 				return "", fmt.Errorf("failed to read fingerprint: %w", err)
 			}
 			ourFingerprint := n.storage.Fingerprint(lower, upper)
@@ -181,6 +181,7 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 			if n.isClient {
 				// notify client of what they have and we don't
 				for _, id := range theirItems {
+					// skip empty strings here because those were marked to be excluded as such in the previous step
 					if id != "" {
 						n.HaveNots <- id
 					}
@@ -226,7 +227,7 @@ func (n *Negentropy) reconcileAux(reader *StringHexReader) (string, error) {
 			remainingFingerprint := n.storage.Fingerprint(upper, n.storage.Size())
 			n.writeBound(fullOutput, InfiniteBound)
 			fullOutput.WriteByte(byte(FingerprintMode))
-			fullOutput.WriteBytes(remainingFingerprint[:])
+			fullOutput.WriteHex(remainingFingerprint)
 
 			break // stop processing further
 		} else {
@@ -286,7 +287,7 @@ func (n *Negentropy) SplitRange(lower, upper int, upperBound Bound, output *Stri
 
 			n.writeBound(output, nextBound)
 			output.WriteByte(byte(FingerprintMode))
-			output.WriteBytes(ourFingerprint[:])
+			output.WriteHex(ourFingerprint)
 		}
 	}
 }

nip77/negentropy/storage.go

@@ -9,5 +9,5 @@ type Storage interface {
 	Range(begin, end int) iter.Seq2[int, Item]
 	FindLowerBound(begin, end int, value Bound) int
 	GetBound(idx int) Bound
-	Fingerprint(begin, end int) [FingerprintSize]byte
+	Fingerprint(begin, end int) string
 }

nip77/negentropy/storage/vector/accumulator.go

@@ -0,0 +1,49 @@
+package vector
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+
+	"github.com/nbd-wtf/go-nostr/nip77/negentropy"
+)
+
+type Accumulator struct {
+	Buf [32 + 8]byte // leave 8 bytes at the end as a slack for use in GetFingerprint append()
+}
+
+func (acc *Accumulator) Reset() {
+	for i := 0; i < 32; i++ {
+		acc.Buf[i] = 0
+	}
+}
+
+func (acc *Accumulator) AddAccumulator(other Accumulator) {
+	acc.AddBytes(other.Buf[:32])
+}
+
+func (acc *Accumulator) AddBytes(other []byte) {
+	var currCarry, nextCarry uint32
+
+	for i := 0; i < 8; i++ {
+		offset := i * 4
+		orig := binary.LittleEndian.Uint32(acc.Buf[offset:])
+		otherV := binary.LittleEndian.Uint32(other[offset:])
+
+		next := orig + currCarry + otherV
+		if next < orig || next < otherV {
+			nextCarry = 1
+		}
+
+		binary.LittleEndian.PutUint32(acc.Buf[offset:32], next&0xFFFFFFFF)
+		currCarry = nextCarry
+		nextCarry = 0
+	}
+}
+
+func (acc *Accumulator) GetFingerprint(n int) string {
+	input := acc.Buf[:32]
+	input = append(input, negentropy.EncodeVarInt(n)...)
+	hash := sha256.Sum256(input)
+	return hex.EncodeToString(hash[:negentropy.FingerprintSize])
+}

nip77/negentropy/storage/vector/vector.go

@@ -13,6 +13,8 @@ import (
 type Vector struct {
 	items  []negentropy.Item
 	sealed bool
+
+	acc Accumulator
 }
 
 func New() *Vector {
@@ -21,14 +23,13 @@ func New() *Vector {
 	}
 }
 
-func (v *Vector) Insert(createdAt nostr.Timestamp, id string) error {
+func (v *Vector) Insert(createdAt nostr.Timestamp, id string) {
 	if len(id) != 64 {
-		return fmt.Errorf("bad id size for added item: expected %d bytes, got %d", 32, len(id)/2)
+		panic(fmt.Errorf("bad id size for added item: expected %d bytes, got %d", 32, len(id)/2))
 	}
 
 	item := negentropy.Item{Timestamp: createdAt, ID: id}
 	v.items = append(v.items, item)
-	return nil
 }
 
 func (v *Vector) Size() int { return len(v.items) }
@@ -63,15 +64,14 @@ func (v *Vector) FindLowerBound(begin, end int, bound negentropy.Bound) int {
 	return begin + idx
 }
 
-func (v *Vector) Fingerprint(begin, end int) [negentropy.FingerprintSize]byte {
+func (v *Vector) Fingerprint(begin, end int) string {
-	var out negentropy.Accumulator
+	v.acc.Reset()
-	out.SetToZero()
 
 	tmp := make([]byte, 32)
 	for _, item := range v.Range(begin, end) {
 		hex.Decode(tmp, []byte(item.ID))
-		out.AddBytes(tmp)
+		v.acc.AddBytes(tmp)
 	}
 
-	return out.GetFingerprint(end - begin)
+	return v.acc.GetFingerprint(end - begin)
 }

nip77/negentropy/testdata/fuzz/FuzzWhatever/08d62a8f20d5938d

@@ -0,0 +1,5 @@
+go test fuzz v1
+uint(165)
+uint(108)
+uint(72)
+uint(54)

nip77/negentropy/testdata/fuzz/FuzzWhatever/162017c93d25a57d

@@ -0,0 +1,5 @@
+go test fuzz v1
+uint(26)
+uint(0)
+uint(58)
+uint(70)

nip77/negentropy/testdata/fuzz/FuzzWhatever/447b9443277df0c9

@@ -0,0 +1,5 @@
+go test fuzz v1
+uint(1)
+uint(17)
+uint(5)
+uint(4044)

nip77/negentropy/testdata/fuzz/FuzzWhatever/77128c6f00c8de14

@@ -0,0 +1,5 @@
+go test fuzz v1
+uint(182)
+uint(303)
+uint(75)
+uint(25)

nip77/negentropy/testdata/fuzz/FuzzWhatever/fe4d1830ff24d72d

@@ -0,0 +1,5 @@
+go test fuzz v1
+uint(17)
+uint(17)
+uint(39)
+uint(4115)

nip77/negentropy/types.go

@@ -2,8 +2,6 @@ package negentropy
 
 import (
 	"cmp"
-	"crypto/sha256"
-	"encoding/binary"
 	"fmt"
 	"strings"
 
@@ -55,51 +53,3 @@ func (b Bound) String() string {
 	}
 	return fmt.Sprintf("Bound<%d:%s>", b.Timestamp, b.ID)
 }
-
-type Accumulator struct {
-	Buf []byte
-}
-
-func (acc *Accumulator) SetToZero() {
-	acc.Buf = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-}
-
-func (acc *Accumulator) AddAccumulator(other Accumulator) {
-	acc.AddBytes(other.Buf)
-}
-
-func (acc *Accumulator) AddBytes(other []byte) {
-	var currCarry, nextCarry uint32
-
-	if len(acc.Buf) < 32 {
-		newBuf := make([]byte, 32)
-		copy(newBuf, acc.Buf)
-		acc.Buf = newBuf
-	}
-
-	for i := 0; i < 8; i++ {
-		offset := i * 4
-		orig := binary.LittleEndian.Uint32(acc.Buf[offset:])
-		otherV := binary.LittleEndian.Uint32(other[offset:])
-
-		next := orig + currCarry + otherV
-		if next < orig || next < otherV {
-			nextCarry = 1
-		}
-
-		binary.LittleEndian.PutUint32(acc.Buf[offset:], next&0xFFFFFFFF)
-		currCarry = nextCarry
-		nextCarry = 0
-	}
-}
-
-func (acc *Accumulator) GetFingerprint(n int) [FingerprintSize]byte {
-	input := acc.Buf[:]
-	input = append(input, encodeVarInt(n)...)
-
-	hash := sha256.Sum256(input)
-
-	var fingerprint [FingerprintSize]byte
-	copy(fingerprint[:], hash[:FingerprintSize])
-	return fingerprint
-}