sdk: make it so replaceable stuff is automatically reloaded from time to time.

sdk/list.go

@@ -60,27 +60,70 @@ func fetchGenericList[I TagItemWithValue](
 
 	events, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{Kinds: []int{actualKind}, Authors: []string{pubkey}})
 	if len(events) != 0 {
+		// ok, we found something locally
 		items := parseItemsFromEventTags(events[0], parseTag)
 		v.Event = events[0]
 		v.Items = items
+
+		// but if we haven't tried fetching from the network recently we should do it
+		lastFetchKey := makeLastFetchKey(actualKind, pubkey)
+		lastFetchData, _ := sys.KVStore.Get(lastFetchKey)
+		if nostr.Now()-decodeTimestamp(lastFetchData) > 7*24*60*60 {
+			newV := tryFetchListFromNetwork(ctx, sys, pubkey, replaceableIndex, parseTag)
+			if newV != nil && newV.Event.CreatedAt > v.Event.CreatedAt {
+				v = *newV
+			}
+
+			// even if we didn't find anything register this because we tried
+			// (and we still have the previous event in our local store)
+			sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
+		}
+
+		// and finally save this to cache
 		cache.SetWithTTL(pubkey, v, time.Hour*6)
 		valueWasJustCached[lockIdx] = true
+
 		return v, true
 	}
 
-	thunk := sys.replaceableLoaders[replaceableIndex].Load(ctx, pubkey)
-	evt, err := thunk()
-	if err == nil {
-		items := parseItemsFromEventTags(evt, parseTag)
-		v.Items = items
-		sys.StoreRelay.Publish(ctx, *evt)
+	if newV := tryFetchListFromNetwork(ctx, sys, pubkey, replaceableIndex, parseTag); newV != nil {
+		v = *newV
+
+		// we'll only save this if we got something which means we found at least one event
+		lastFetchKey := makeLastFetchKey(actualKind, pubkey)
+		sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
 	}
+
+	// save cache even if we didn't get anything
 	cache.SetWithTTL(pubkey, v, time.Hour*6)
 	valueWasJustCached[lockIdx] = true
 
 	return v, false
 }
 
+func tryFetchListFromNetwork[I TagItemWithValue](
+	ctx context.Context,
+	sys *System,
+	pubkey string,
+	replaceableIndex replaceableIndex,
+	parseTag func(nostr.Tag) (I, bool),
+) *GenericList[I] {
+	thunk := sys.replaceableLoaders[replaceableIndex].Load(ctx, pubkey)
+	evt, err := thunk()
+	if err != nil {
+		return nil
+	}
+
+	v := &GenericList[I]{
+		PubKey: pubkey,
+		Event:  evt,
+		Items:  parseItemsFromEventTags(evt, parseTag),
+	}
+	sys.StoreRelay.Publish(ctx, *evt)
+
+	return v
+}
+
 func parseItemsFromEventTags[I TagItemWithValue](
 	evt *nostr.Event,
 	parseTag func(nostr.Tag) (I, bool),

sdk/loader_helpers.go

@@ -0,0 +1,36 @@
+package sdk
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var kvStoreLastFetchPrefix = byte('f')
+
+func makeLastFetchKey(kind int, pubkey string) []byte {
+	buf := make([]byte, 1+5+32)
+	buf[0] = kvStoreLastFetchPrefix
+	binary.LittleEndian.PutUint32(buf[1:], uint32(kind))
+	hex.Decode(buf[5:], []byte(pubkey))
+	return buf
+}
+
+// encodeTimestamp encodes a unix timestamp as 4 bytes
+func encodeTimestamp(t nostr.Timestamp) []byte {
+	b := make([]byte, 4)
+	binary.BigEndian.PutUint32(b, uint32(t))
+	return b
+}
+
+// decodeTimestamp decodes a 4-byte timestamp into unix seconds
+func decodeTimestamp(b []byte) nostr.Timestamp {
+	return nostr.Timestamp(binary.BigEndian.Uint32(b))
+}
+
+// shouldRefreshFromNetwork checks if we should try fetching from network
+func shouldRefreshFromNetwork(lastFetchData []byte) bool {
+	lastFetch := decodeTimestamp(lastFetchData)
+	return nostr.Now()-lastFetch > 7*24*60*60
+}

sdk/metadata.go

@@ -106,32 +106,42 @@ func (sys *System) FetchProfileMetadata(ctx context.Context, pubkey string) (pm
 
 	res, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{Kinds: []int{0}, Authors: []string{pubkey}})
 	if len(res) != 0 {
-		if m, err := ParseMetadata(res[0]); err == nil {
-			m.PubKey = pubkey
-			m.Event = res[0]
-			sys.MetadataCache.SetWithTTL(pubkey, m, time.Hour*6)
-			return m
+		// ok, we found something locally
+		pm, _ = ParseMetadata(res[0])
+		pm.PubKey = pubkey
+		pm.Event = res[0]
+
+		// but if we haven't tried fetching from the network recently we should do it
+		lastFetchKey := makeLastFetchKey(0, pubkey)
+		lastFetchData, _ := sys.KVStore.Get(lastFetchKey)
+		if nostr.Now()-decodeTimestamp(lastFetchData) > 7*24*60*60 {
+			newM := sys.tryFetchMetadataFromNetwork(ctx, pubkey)
+			if newM != nil && newM.Event.CreatedAt > pm.Event.CreatedAt {
+				pm = *newM
+			}
+
+			// even if we didn't find anything register this because we tried
+			// (and we still have the previous event in our local store)
+			sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
 		}
-	}
 
-	pm.PubKey = pubkey
-
-	thunk0 := sys.replaceableLoaders[kind_0].Load(ctx, pubkey)
-	evt, err := thunk0()
-	if err == nil {
-		pm, _ = ParseMetadata(evt)
-
-		// save on store even if the metadata json is malformed
-		if pm.Event != nil {
-			sys.StoreRelay.Publish(ctx, *pm.Event)
-		}
-	}
-
-	// save on cache even if the metadata isn't found (unless the context was canceled)
-	if err == nil || err != context.Canceled {
+		// and finally save this to cache
 		sys.MetadataCache.SetWithTTL(pubkey, pm, time.Hour*6)
+
+		return pm
 	}
 
+	if newM := sys.tryFetchMetadataFromNetwork(ctx, pubkey); newM != nil {
+		pm = *newM
+
+		// we'll only save this if we got something which means we found at least one event
+		lastFetchKey := makeLastFetchKey(0, pubkey)
+		sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
+	}
+
+	// save cache even if we didn't get anything
+	sys.MetadataCache.SetWithTTL(pubkey, pm, time.Hour*6)
+
 	return pm
 }
 
@@ -159,6 +169,25 @@ func (sys *System) FetchUserEvents(ctx context.Context, filter nostr.Filter) (ma
 	return results, nil
 }
 
+func (sys *System) tryFetchMetadataFromNetwork(ctx context.Context, pubkey string) *ProfileMetadata {
+	thunk0 := sys.replaceableLoaders[kind_0].Load(ctx, pubkey)
+	evt, err := thunk0()
+	if err != nil {
+		return nil
+	}
+
+	pm, err := ParseMetadata(evt)
+	if err != nil {
+		return nil
+	}
+
+	pm.PubKey = pubkey
+	pm.Event = evt
+	sys.StoreRelay.Publish(ctx, *evt)
+	sys.MetadataCache.SetWithTTL(pubkey, pm, time.Hour*6)
+	return &pm
+}
+
 func ParseMetadata(event *nostr.Event) (meta ProfileMetadata, err error) {
 	if event.Kind != 0 {
 		err = fmt.Errorf("event %s is kind %d, not 0", event.ID, event.Kind)

sdk/set.go

@@ -49,29 +49,71 @@ func fetchGenericSets[I TagItemWithValue](
 
 	events, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{Kinds: []int{actualKind}, Authors: []string{pubkey}})
 	if len(events) != 0 {
+		// ok, we found something locally
 		sets := parseSetsFromEvents(events, parseTag)
 		v.Events = events
 		v.Sets = sets
+
+		// but if we haven't tried fetching from the network recently we should do it
+		lastFetchKey := makeLastFetchKey(actualKind, pubkey)
+		lastFetchData, _ := sys.KVStore.Get(lastFetchKey)
+		if nostr.Now()-decodeTimestamp(lastFetchData) > 7*24*60*60 {
+			newV := tryFetchSetsFromNetwork(ctx, sys, pubkey, addressableIndex, parseTag)
+
+			// unlike for lists, when fetching sets we will blindly trust whatever we get from the network
+			v = *newV
+
+			// even if we didn't find anything register this because we tried
+			// (and we still have the previous event in our local store)
+			sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
+		}
+
+		// and finally save this to cache
 		cache.SetWithTTL(pubkey, v, time.Hour*6)
 		valueWasJustCached[lockIdx] = true
+
 		return v, true
 	}
 
-	thunk := sys.addressableLoaders[addressableIndex].Load(ctx, pubkey)
-	events, err := thunk()
-	if err == nil {
-		sets := parseSetsFromEvents(events, parseTag)
-		v.Sets = sets
-		for _, evt := range events {
-			sys.StoreRelay.Publish(ctx, *evt)
-		}
+	if newV := tryFetchSetsFromNetwork(ctx, sys, pubkey, addressableIndex, parseTag); newV != nil {
+		v = *newV
+
+		// we'll only save this if we got something which means we found at least one event
+		lastFetchKey := makeLastFetchKey(actualKind, pubkey)
+		sys.KVStore.Set(lastFetchKey, encodeTimestamp(nostr.Now()))
 	}
+
+	// save cache even if we didn't get anything
 	cache.SetWithTTL(pubkey, v, time.Hour*6)
 	valueWasJustCached[lockIdx] = true
 
 	return v, false
 }
 
+func tryFetchSetsFromNetwork[I TagItemWithValue](
+	ctx context.Context,
+	sys *System,
+	pubkey string,
+	addressableIndex addressableIndex,
+	parseTag func(nostr.Tag) (I, bool),
+) *GenericSets[I] {
+	thunk := sys.addressableLoaders[addressableIndex].Load(ctx, pubkey)
+	events, err := thunk()
+	if err != nil {
+		return nil
+	}
+
+	v := &GenericSets[I]{
+		PubKey: pubkey,
+		Events: events,
+		Sets:   parseSetsFromEvents(events, parseTag),
+	}
+	for _, evt := range events {
+		sys.StoreRelay.Publish(ctx, *evt)
+	}
+	return v
+}
+
 func parseSetsFromEvents[I TagItemWithValue](
 	events []*nostr.Event,
 	parseTag func(nostr.Tag) (I, bool),