move nostr-sdk repository into here because why not?

sdk/hints/interface.go

@@ -0,0 +1,8 @@
+package hints
+
+import "github.com/nbd-wtf/go-nostr"
+
+type HintsDB interface {
+	TopN(pubkey string, n int) []string
+	Save(pubkey string, relay string, key HintKey, score nostr.Timestamp)
+}

sdk/hints/keys.go

@@ -0,0 +1,49 @@
+package hints
+
+import "github.com/nbd-wtf/go-nostr"
+
+const END_OF_WORLD nostr.Timestamp = 2208999600 // 2040-01-01
+
+type HintKey int
+
+const (
+	LastFetchAttempt HintKey = iota
+	MostRecentEventFetched
+	LastInRelayList
+	LastInTag
+	LastInNprofile
+	LastInNevent
+	LastInNIP05
+)
+
+var KeyBasePoints = [7]int64{
+	-500, // attempting has negative power because it may fail
+	700,  // when it succeeds that should cancel the negative effect of trying
+	350,  // a relay list is a very strong indicator
+	5,    // tag hints are often autogenerated so we don't care very much about them (that may change)
+	22,   // it feels like people take nprofiles slightly more seriously so we value these a bit more
+	8,    // these are also not often too bad
+	7,    // nip05 hints should be a strong indicator, although in practice they're kinda bad
+}
+
+func (hk HintKey) BasePoints() int64 { return KeyBasePoints[hk] }
+
+func (hk HintKey) String() string {
+	switch hk {
+	case LastFetchAttempt:
+		return "last_fetch_attempt"
+	case MostRecentEventFetched:
+		return "most_recent_event_fetched"
+	case LastInRelayList:
+		return "last_in_relay_list"
+	case LastInTag:
+		return "last_in_tag"
+	case LastInNprofile:
+		return "last_in_nprofile"
+	case LastInNevent:
+		return "last_in_nevent"
+	case LastInNIP05:
+		return "last_in_nip05"
+	}
+	return "<unexpected>"
+}

sdk/hints/memory/memory_hints.go

@@ -0,0 +1,142 @@
+package memory
+
+import (
+	"fmt"
+	"math"
+	"slices"
+	"sync"
+
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/sdk/hints"
+)
+
+var _ hints.HintsDB = (*HintDB)(nil)
+
+type HintDB struct {
+	RelayBySerial         []string
+	OrderedRelaysByPubKey map[string]RelaysForPubKey
+
+	sync.Mutex
+}
+
+func NewHintDB() *HintDB {
+	return &HintDB{
+		RelayBySerial:         make([]string, 0, 100),
+		OrderedRelaysByPubKey: make(map[string]RelaysForPubKey, 100),
+	}
+}
+
+func (db *HintDB) Save(pubkey string, relay string, key hints.HintKey, ts nostr.Timestamp) {
+	now := nostr.Now()
+	// this is used for calculating what counts as a usable hint
+	threshold := (now - 60*60*24*180)
+	if threshold < 0 {
+		threshold = 0
+	}
+
+	relayIndex := slices.Index(db.RelayBySerial, relay)
+	if relayIndex == -1 {
+		relayIndex = len(db.RelayBySerial)
+		db.RelayBySerial = append(db.RelayBySerial, relay)
+	}
+
+	db.Lock()
+	defer db.Unlock()
+	// fmt.Println(" ", relay, "index", relayIndex, "--", "adding", hints.HintKey(key).String(), ts)
+
+	rfpk, _ := db.OrderedRelaysByPubKey[pubkey]
+
+	entries := rfpk.Entries
+
+	entryIndex := slices.IndexFunc(entries, func(re RelayEntry) bool { return re.Relay == relayIndex })
+	if entryIndex == -1 {
+		// we don't have an entry for this relay, so add one
+		entryIndex = len(entries)
+
+		entry := RelayEntry{
+			Relay: relayIndex,
+		}
+		entry.Timestamps[key] = ts
+
+		entries = append(entries, entry)
+	} else {
+		// just update this entry
+		if entries[entryIndex].Timestamps[key] < ts {
+			entries[entryIndex].Timestamps[key] = ts
+		} else {
+			// no need to update anything
+			return
+		}
+	}
+
+	rfpk.Entries = entries
+
+	db.OrderedRelaysByPubKey[pubkey] = rfpk
+}
+
+func (db *HintDB) TopN(pubkey string, n int) []string {
+	db.Lock()
+	defer db.Unlock()
+
+	urls := make([]string, 0, n)
+	if rfpk, ok := db.OrderedRelaysByPubKey[pubkey]; ok {
+		// sort everything from scratch
+		slices.SortFunc(rfpk.Entries, func(a, b RelayEntry) int {
+			return int(b.Sum() - a.Sum())
+		})
+
+		for i, re := range rfpk.Entries {
+			urls = append(urls, db.RelayBySerial[re.Relay])
+			if i+1 == n {
+				break
+			}
+		}
+	}
+	return urls
+}
+
+func (db *HintDB) PrintScores() {
+	db.Lock()
+	defer db.Unlock()
+
+	fmt.Println("= print scores")
+	for pubkey, rfpk := range db.OrderedRelaysByPubKey {
+		fmt.Println("== relay scores for", pubkey)
+		for i, re := range rfpk.Entries {
+			fmt.Printf("  %3d :: %30s (%3d) ::> %12d\n", i, db.RelayBySerial[re.Relay], re.Relay, re.Sum())
+		}
+	}
+}
+
+type RelaysForPubKey struct {
+	Entries []RelayEntry
+}
+
+type RelayEntry struct {
+	Relay      int
+	Timestamps [8]nostr.Timestamp
+}
+
+func (re RelayEntry) Sum() int64 {
+	now := nostr.Now() + 24*60*60
+	var sum int64
+	for i, ts := range re.Timestamps {
+		if ts == 0 {
+			continue
+		}
+
+		hk := hints.HintKey(i)
+		divisor := int64(now - ts)
+		if divisor == 0 {
+			divisor = 1
+		} else {
+			divisor = int64(math.Pow(float64(divisor), 1.3))
+		}
+
+		multiplier := hk.BasePoints()
+		value := multiplier * 10000000000 / divisor
+		// fmt.Println("   ", i, "value:", value)
+		sum += value
+	}
+	return sum
+}

sdk/hints/memory/memory_test.go

@@ -0,0 +1,143 @@
+package memory
+
+import (
+	"testing"
+	"time"
+
+	"github.com/nbd-wtf/go-nostr"
+	"github.com/nbd-wtf/go-nostr/sdk/hints"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRelayPicking(t *testing.T) {
+	hdb := NewHintDB()
+
+	const key1 = "0000000000000000000000000000000000000000000000000000000000000001"
+	const key2 = "0000000000000000000000000000000000000000000000000000000000000002"
+	const key3 = "0000000000000000000000000000000000000000000000000000000000000003"
+	const key4 = "0000000000000000000000000000000000000000000000000000000000000004"
+	const relayA = "wss://aaa.com"
+	const relayB = "wss://bbb.online"
+	const relayC = "wss://ccc.technology"
+
+	hour := nostr.Timestamp((time.Hour).Seconds())
+	day := hour * 24
+
+	// key1: finding out
+	// add some random parameters things and see what we get
+	hdb.Save(key1, relayA, hints.LastInTag, nostr.Now()-60*hour)
+	hdb.Save(key1, relayB, hints.LastInRelayList, nostr.Now()-day*10)
+	hdb.Save(key1, relayB, hints.LastInNevent, nostr.Now()-day*30)
+	hdb.Save(key1, relayA, hints.LastInNprofile, nostr.Now()-hour*10)
+	hdb.PrintScores()
+
+	require.Equal(t, []string{relayB, relayA}, hdb.TopN(key1, 3))
+
+	hdb.Save(key1, relayA, hints.LastFetchAttempt, nostr.Now()-5*hour)
+	hdb.Save(key1, relayC, hints.LastInNIP05, nostr.Now()-5*hour)
+	hdb.PrintScores()
+
+	require.Equal(t, []string{relayB, relayC, relayA}, hdb.TopN(key1, 3))
+
+	hdb.Save(key1, relayC, hints.LastInTag, nostr.Now()-5*hour)
+	hdb.Save(key1, relayC, hints.LastFetchAttempt, nostr.Now()-5*hour)
+	hdb.PrintScores()
+
+	require.Equal(t, []string{relayB, relayA, relayC}, hdb.TopN(key1, 3))
+
+	hdb.Save(key1, relayA, hints.MostRecentEventFetched, nostr.Now()-day*60)
+	hdb.PrintScores()
+
+	require.Equal(t, []string{relayB, relayA, relayC}, hdb.TopN(key1, 3))
+
+	// now let's try a different thing for key2
+	// key2 has a relay list with A and B
+	hdb.Save(key2, relayA, hints.LastInRelayList, nostr.Now()-day*25)
+	hdb.Save(key2, relayB, hints.LastInRelayList, nostr.Now()-day*25)
+
+	// but it's old, recently we only see hints for relay C
+	hdb.Save(key2, relayC, hints.LastInTag, nostr.Now()-5*hour)
+	hdb.Save(key2, relayC, hints.LastInNIP05, nostr.Now()-5*hour)
+	hdb.Save(key2, relayC, hints.LastInNevent, nostr.Now()-5*hour)
+	hdb.Save(key2, relayC, hints.LastInNprofile, nostr.Now()-5*hour)
+
+	// at this point we just barely see C coming first
+	hdb.PrintScores()
+	require.Equal(t, []string{relayC, relayA, relayB}, hdb.TopN(key2, 3))
+
+	// yet a different thing for key3
+	// it doesn't have relay lists published because it's banned everywhere
+	// all it has are references to its posts from others
+	hdb.Save(key3, relayA, hints.LastInTag, nostr.Now()-day*2)
+	hdb.Save(key3, relayB, hints.LastInNevent, nostr.Now()-day)
+	hdb.Save(key3, relayB, hints.LastInTag, nostr.Now()-day)
+	hdb.PrintScores()
+	require.Equal(t, []string{relayB, relayA}, hdb.TopN(key3, 3))
+
+	// we try to fetch events for key3 and we get a very recent one for relay A, an older for relay B
+	hdb.Save(key3, relayA, hints.LastFetchAttempt, nostr.Now()-5*hour)
+	hdb.Save(key3, relayA, hints.MostRecentEventFetched, nostr.Now()-day)
+	hdb.Save(key3, relayB, hints.LastFetchAttempt, nostr.Now()-5*hour)
+	hdb.Save(key3, relayB, hints.MostRecentEventFetched, nostr.Now()-day*30)
+	hdb.PrintScores()
+	require.Equal(t, []string{relayA, relayB}, hdb.TopN(key3, 3))
+
+	// for key4 we'll try the alex jones case
+	// key4 used to publish normally to a bunch of big relays until it got banned
+	// then it started publishing only to its personal relay
+	// how long until clients realize that?
+	banDate := nostr.Now() - day*10
+	hdb.Save(key4, relayA, hints.LastInRelayList, banDate)
+	hdb.Save(key4, relayA, hints.LastFetchAttempt, banDate)
+	hdb.Save(key4, relayA, hints.MostRecentEventFetched, banDate)
+	hdb.Save(key4, relayA, hints.LastInNprofile, banDate+8*day)
+	hdb.Save(key4, relayA, hints.LastInNIP05, banDate+5*day)
+	hdb.Save(key4, relayB, hints.LastInRelayList, banDate)
+	hdb.Save(key4, relayB, hints.LastFetchAttempt, banDate)
+	hdb.Save(key4, relayB, hints.MostRecentEventFetched, banDate)
+	hdb.Save(key4, relayB, hints.LastInNevent, banDate+5*day)
+	hdb.Save(key4, relayB, hints.LastInNIP05, banDate+8*day)
+	hdb.Save(key4, relayB, hints.LastInNprofile, banDate+5*day)
+	hdb.PrintScores()
+	require.Equal(t, []string{relayA, relayB}, hdb.TopN(key4, 3))
+
+	// information about the new relay starts to spread through relay hints in tags only
+	hdb.Save(key4, relayC, hints.LastInTag, nostr.Now()-5*day)
+	hdb.Save(key4, relayC, hints.LastInTag, nostr.Now()-5*day)
+	hdb.Save(key4, relayC, hints.LastInNevent, nostr.Now()-5*day)
+	hdb.Save(key4, relayC, hints.LastInNIP05, nostr.Now()-5*day)
+
+	// as long as we see one tag hint the new relay will already be in our map
+	hdb.PrintScores()
+	require.Equal(t, []string{relayA, relayB, relayC}, hdb.TopN(key4, 3))
+
+	// client tries to fetch stuff from the old relays, but gets nothing new
+	hdb.Save(key4, relayA, hints.LastFetchAttempt, nostr.Now()-5*hour)
+	hdb.Save(key4, relayB, hints.LastFetchAttempt, nostr.Now()-5*hour)
+
+	// which is enough for us to transition to the new relay as the toppermost of the uppermost
+	hdb.PrintScores()
+	require.Equal(t, []string{relayC, relayA, relayB}, hdb.TopN(key4, 3))
+
+	// what if the big relays are attempting to game this algorithm by allowing some of our
+	// events from time to time while still shadowbanning us?
+	hdb.Save(key4, relayA, hints.MostRecentEventFetched, nostr.Now()-5*hour)
+	hdb.Save(key4, relayB, hints.MostRecentEventFetched, nostr.Now()-5*hour)
+	hdb.PrintScores()
+	require.Equal(t, []string{relayA, relayB, relayC}, hdb.TopN(key4, 3))
+
+	// we'll need overwhelming force from the third relay
+	// (actually just a relay list with just its name in it will be enough)
+	hdb.Save(key4, relayC, hints.LastFetchAttempt, nostr.Now()-5*hour)
+	hdb.Save(key4, relayC, hints.MostRecentEventFetched, nostr.Now()-6*hour)
+	hdb.Save(key4, relayC, hints.LastInRelayList, nostr.Now()-6*hour)
+	hdb.PrintScores()
+	require.Equal(t, []string{relayC, relayA, relayB}, hdb.TopN(key4, 3))
+
+	//
+	//
+	// things remain the same for key1, key2 and key3
+	require.Equal(t, []string{relayC, relayA}, hdb.TopN(key2, 2))
+	require.Equal(t, []string{relayB, relayA, relayC}, hdb.TopN(key1, 3))
+	require.Equal(t, []string{relayA, relayB}, hdb.TopN(key3, 3))
+}