Uses per subscription EOSE to avoid downloading everything again when Resuming the app.

2025-09-26 02:46:41 +02:00 · 2023-03-31 12:26:13 -04:00
parent 6884d5e8dd
commit bd62736002
9 changed files with 90 additions and 18 deletions
--- a/app/src/main/java/com/vitorpamplona/amethyst/model/Note.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/model/Note.kt
@@ -3,6 +3,7 @@ package com.vitorpamplona.amethyst.model
 import androidx.lifecycle.LiveData
 import com.vitorpamplona.amethyst.service.NostrSingleEventDataSource
 import com.vitorpamplona.amethyst.service.model.*
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.Relay
 import com.vitorpamplona.amethyst.ui.components.BundledUpdate
 import com.vitorpamplona.amethyst.ui.note.toShortenHex
@@ -46,7 +47,7 @@ open class Note(val idHex: String) {
    var relays = setOf<String>()
        private set

-    var lastReactionsDownloadTime: Map<String, Long> = emptyMap()
+    var lastReactionsDownloadTime: Map<String, EOSETime> = emptyMap()

    fun id() = Hex.decode(idHex)
    open fun idNote() = id().toNote()
--- a/app/src/main/java/com/vitorpamplona/amethyst/model/User.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/model/User.kt
@@ -7,6 +7,7 @@ import com.vitorpamplona.amethyst.service.model.ContactListEvent
 import com.vitorpamplona.amethyst.service.model.LnZapEvent
 import com.vitorpamplona.amethyst.service.model.MetadataEvent
 import com.vitorpamplona.amethyst.service.model.ReportEvent
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.Relay
 import com.vitorpamplona.amethyst.ui.components.BundledUpdate
 import com.vitorpamplona.amethyst.ui.note.toShortenHex
@@ -31,7 +32,7 @@ class User(val pubkeyHex: String) {
    var reports = mapOf<User, Set<Note>>()
        private set

-    var latestEOSEs: Map<String, Long> = emptyMap()
+    var latestEOSEs: Map<String, EOSETime> = emptyMap()

    var zaps = mapOf<Note, Note?>()
        private set
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/NostrAccountDataSource.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/NostrAccountDataSource.kt
@@ -12,6 +12,7 @@ import com.vitorpamplona.amethyst.service.model.ReactionEvent
 import com.vitorpamplona.amethyst.service.model.ReportEvent
 import com.vitorpamplona.amethyst.service.model.RepostEvent
 import com.vitorpamplona.amethyst.service.model.TextNoteEvent
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.FeedType
 import com.vitorpamplona.amethyst.service.relays.JsonFilter
 import com.vitorpamplona.amethyst.service.relays.TypedFilter
@@ -19,6 +20,8 @@ import com.vitorpamplona.amethyst.service.relays.TypedFilter
 object NostrAccountDataSource : NostrDataSource("AccountData") {
    lateinit var account: Account

+    var latestEOSEs: Map<String, EOSETime> = emptyMap()
+
    fun createAccountContactListFilter(): TypedFilter {
        return TypedFilter(
            types = FeedType.values().toSet(),
@@ -68,7 +71,8 @@ object NostrAccountDataSource : NostrDataSource("AccountData") {
            types = FeedType.values().toSet(),
            filter = JsonFilter(
                kinds = listOf(ReportEvent.kind),
-                authors = listOf(account.userProfile().pubkeyHex)
+                authors = listOf(account.userProfile().pubkeyHex),
+                since = latestEOSEs
            )
        )
    }
@@ -86,11 +90,19 @@ object NostrAccountDataSource : NostrDataSource("AccountData") {
                BadgeAwardEvent.kind
            ),
            tags = mapOf("p" to listOf(account.userProfile().pubkeyHex)),
-            limit = 200
+            limit = 400,
+            since = latestEOSEs
        )
    )

-    val accountChannel = requestNewChannel()
+    val accountChannel = requestNewChannel { time, relayUrl ->
+        val eose = latestEOSEs[relayUrl]
+        if (eose == null) {
+            latestEOSEs = latestEOSEs + Pair(relayUrl, EOSETime(time))
+        } else {
+            eose.time = time
+        }
+    }

    override fun updateChannelFilters() {
        // gets everthing about the user logged in
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/NostrChatroomListDataSource.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/NostrChatroomListDataSource.kt
@@ -5,6 +5,7 @@ import com.vitorpamplona.amethyst.service.model.ChannelCreateEvent
 import com.vitorpamplona.amethyst.service.model.ChannelMessageEvent
 import com.vitorpamplona.amethyst.service.model.ChannelMetadataEvent
 import com.vitorpamplona.amethyst.service.model.PrivateDmEvent
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.FeedType
 import com.vitorpamplona.amethyst.service.relays.JsonFilter
 import com.vitorpamplona.amethyst.service.relays.TypedFilter
@@ -12,11 +13,14 @@ import com.vitorpamplona.amethyst.service.relays.TypedFilter
 object NostrChatroomListDataSource : NostrDataSource("MailBoxFeed") {
    lateinit var account: Account

+    var latestEOSEs: Map<String, EOSETime> = emptyMap()
+
    fun createMessagesToMeFilter() = TypedFilter(
        types = setOf(FeedType.PRIVATE_DMS),
        filter = JsonFilter(
            kinds = listOf(PrivateDmEvent.kind),
-            tags = mapOf("p" to listOf(account.userProfile().pubkeyHex))
+            tags = mapOf("p" to listOf(account.userProfile().pubkeyHex)),
+            since = latestEOSEs
        )
    )

@@ -24,7 +28,8 @@ object NostrChatroomListDataSource : NostrDataSource("MailBoxFeed") {
        types = setOf(FeedType.PRIVATE_DMS),
        filter = JsonFilter(
            kinds = listOf(PrivateDmEvent.kind),
-            authors = listOf(account.userProfile().pubkeyHex)
+            authors = listOf(account.userProfile().pubkeyHex),
+            since = latestEOSEs
        )
    )

@@ -32,7 +37,8 @@ object NostrChatroomListDataSource : NostrDataSource("MailBoxFeed") {
        types = setOf(FeedType.PUBLIC_CHATS),
        filter = JsonFilter(
            kinds = listOf(ChannelCreateEvent.kind, ChannelMetadataEvent.kind),
-            authors = listOf(account.userProfile().pubkeyHex)
+            authors = listOf(account.userProfile().pubkeyHex),
+            since = latestEOSEs
        )
    )

@@ -40,7 +46,8 @@ object NostrChatroomListDataSource : NostrDataSource("MailBoxFeed") {
        types = FeedType.values().toSet(), // Metadata comes from any relay
        filter = JsonFilter(
            kinds = listOf(ChannelCreateEvent.kind),
-            ids = account.followingChannels.toList()
+            ids = account.followingChannels.toList(),
+            since = latestEOSEs
        )
    )

@@ -64,13 +71,21 @@ object NostrChatroomListDataSource : NostrDataSource("MailBoxFeed") {
                filter = JsonFilter(
                    kinds = listOf(ChannelMessageEvent.kind),
                    tags = mapOf("e" to listOf(it)),
+                    since = latestEOSEs,
                    limit = 25 // Remember to consider spam that is being removed from the UI
                )
            )
        }
    }

-    val chatroomListChannel = requestNewChannel()
+    val chatroomListChannel = requestNewChannel() { time, relayUrl ->
+        val eose = latestEOSEs[relayUrl]
+        if (eose == null) {
+            latestEOSEs = latestEOSEs + Pair(relayUrl, EOSETime(time))
+        } else {
+            eose.time = time
+        }
+    }

    override fun updateChannelFilters() {
        val list = listOf(
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/NostrHomeDataSource.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/NostrHomeDataSource.kt
@@ -4,6 +4,7 @@ import com.vitorpamplona.amethyst.model.Account
 import com.vitorpamplona.amethyst.model.UserState
 import com.vitorpamplona.amethyst.service.model.LongTextNoteEvent
 import com.vitorpamplona.amethyst.service.model.TextNoteEvent
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.FeedType
 import com.vitorpamplona.amethyst.service.relays.JsonFilter
 import com.vitorpamplona.amethyst.service.relays.TypedFilter
@@ -15,6 +16,8 @@ import kotlinx.coroutines.launch
 object NostrHomeDataSource : NostrDataSource("HomeFeed") {
    lateinit var account: Account

+    var latestEOSEs: Map<String, EOSETime> = emptyMap()
+
    private val cacheListener: (UserState) -> Unit = {
        invalidateFilters()
    }
@@ -53,7 +56,8 @@ object NostrHomeDataSource : NostrDataSource("HomeFeed") {
            filter = JsonFilter(
                kinds = listOf(TextNoteEvent.kind, LongTextNoteEvent.kind),
                authors = followSet,
-                limit = 400
+                limit = 400,
+                since = latestEOSEs
            )
        )
    }
@@ -72,12 +76,20 @@ object NostrHomeDataSource : NostrDataSource("HomeFeed") {
                        listOf(it, it.lowercase(), it.uppercase(), it.capitalize())
                    }.flatten()
                ),
-                limit = 100
+                limit = 100,
+                since = latestEOSEs
            )
        )
    }

-    val followAccountChannel = requestNewChannel()
+    val followAccountChannel = requestNewChannel() { time, relayUrl ->
+        val eose = latestEOSEs[relayUrl]
+        if (eose == null) {
+            latestEOSEs = latestEOSEs + Pair(relayUrl, EOSETime(time))
+        } else {
+            eose.time = time
+        }
+    }

    override fun updateChannelFilters() {
        followAccountChannel.typedFilters = listOfNotNull(createFollowAccountsFilter(), createFollowTagsFilter()).ifEmpty { null }
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/NostrSingleEventDataSource.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/NostrSingleEventDataSource.kt
@@ -15,6 +15,7 @@ import com.vitorpamplona.amethyst.service.model.ReactionEvent
 import com.vitorpamplona.amethyst.service.model.ReportEvent
 import com.vitorpamplona.amethyst.service.model.RepostEvent
 import com.vitorpamplona.amethyst.service.model.TextNoteEvent
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.FeedType
 import com.vitorpamplona.amethyst.service.relays.JsonFilter
 import com.vitorpamplona.amethyst.service.relays.TypedFilter
@@ -136,8 +137,14 @@ object NostrSingleEventDataSource : NostrDataSource("SingleEventFeed") {

    val singleEventChannel = requestNewChannel { time, relayUrl ->
        eventsToWatch.forEach {
-            it.lastReactionsDownloadTime = it.lastReactionsDownloadTime + Pair(relayUrl, time)
+            val eose = it.lastReactionsDownloadTime[relayUrl]
+            if (eose == null) {
+                it.lastReactionsDownloadTime = it.lastReactionsDownloadTime + Pair(relayUrl, EOSETime(time))
+            } else {
+                eose.time = time
+            }
        }
+
        // Many relays operate with limits in the amount of filters.
        // As information comes, the filters will be rotated to get more data.
        invalidateFilters()
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/NostrSingleUserDataSource.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/NostrSingleUserDataSource.kt
@@ -3,6 +3,7 @@ package com.vitorpamplona.amethyst.service
 import com.vitorpamplona.amethyst.model.User
 import com.vitorpamplona.amethyst.service.model.MetadataEvent
 import com.vitorpamplona.amethyst.service.model.ReportEvent
+import com.vitorpamplona.amethyst.service.relays.EOSETime
 import com.vitorpamplona.amethyst.service.relays.FeedType
 import com.vitorpamplona.amethyst.service.relays.JsonFilter
 import com.vitorpamplona.amethyst.service.relays.TypedFilter
@@ -42,8 +43,14 @@ object NostrSingleUserDataSource : NostrDataSource("SingleUserFeed") {

    val userChannel = requestNewChannel() { time, relayUrl ->
        usersToWatch.forEach {
-            it.latestEOSEs = it.latestEOSEs + Pair(relayUrl, time)
+            val eose = it.latestEOSEs[relayUrl]
+            if (eose == null) {
+                it.latestEOSEs = it.latestEOSEs + Pair(relayUrl, EOSETime(time))
+            } else {
+                eose.time = time
+            }
        }
+
        // Many relays operate with limits in the amount of filters.
        // As information comes, the filters will be rotated to get more data.
        invalidateFilters()
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/relays/JsonFilter.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/relays/JsonFilter.kt
@@ -6,12 +6,14 @@ import com.google.gson.JsonArray
 import com.google.gson.JsonObject
 import java.util.*

+class EOSETime(var time: Long)
+
 class JsonFilter(
    val ids: List<String>? = null,
    val authors: List<String>? = null,
    val kinds: List<Int>? = null,
    val tags: Map<String, List<String>>? = null,
-    val since: Map<String, Long>? = null,
+    val since: Map<String, EOSETime>? = null,
    val until: Long? = null,
    val limit: Int? = null,
    val search: String? = null
@@ -37,7 +39,7 @@ class JsonFilter(
                if (forRelay != null) {
                    val relaySince = get(forRelay)
                    if (relaySince != null) {
-                        jsonObject.addProperty("since", relaySince)
+                        jsonObject.addProperty("since", relaySince.time)
                    }
                } else {
                    val jsonObjectSince = JsonObject()
--- a/app/src/main/java/com/vitorpamplona/amethyst/service/relays/Relay.kt
+++ b/app/src/main/java/com/vitorpamplona/amethyst/service/relays/Relay.kt
@@ -40,6 +40,8 @@ class Relay(

    var closingTime = 0L

+    var afterEOSE = false
+
    fun register(listener: Listener) {
        listeners = listeners.plus(listener)
    }
@@ -74,6 +76,7 @@ class Relay(
            val listener = object : WebSocketListener() {

                override fun onOpen(webSocket: WebSocket, response: Response) {
+                    afterEOSE = false
                    isReady = true
                    ping = response.receivedResponseAtMillis - response.sentRequestAtMillis
                    // Log.w("Relay", "Relay OnOpen, Loading All subscriptions $url")
@@ -89,12 +92,19 @@ class Relay(
                        val msg = Event.gson.fromJson(text, JsonElement::class.java).asJsonArray
                        val type = msg[0].asString
                        val channel = msg[1].asString
+
                        when (type) {
                            "EVENT" -> {
                                // Log.w("Relay", "Relay onEVENT $url, $channel")
-                                listeners.forEach { it.onEvent(this@Relay, channel, Event.fromJson(msg[2], Client.lenient)) }
+                                listeners.forEach {
+                                    it.onEvent(this@Relay, channel, Event.fromJson(msg[2], Client.lenient))
+                                    if (afterEOSE) {
+                                        it.onRelayStateChange(this@Relay, Type.EOSE, channel)
+                                    }
+                                }
                            }
                            "EOSE" -> listeners.forEach {
+                                afterEOSE = true
                                // Log.w("Relay", "Relay onEOSE $url, $channel")
                                it.onRelayStateChange(this@Relay, Type.EOSE, channel)
                            }
@@ -136,6 +146,7 @@ class Relay(
                override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
                    socket = null
                    isReady = false
+                    afterEOSE = false
                    closingTime = Date().time / 1000
                    listeners.forEach { it.onRelayStateChange(this@Relay, Type.DISCONNECT, null) }
                }
@@ -147,6 +158,7 @@ class Relay(
                    // Failures disconnect the relay.
                    socket = null
                    isReady = false
+                    afterEOSE = false
                    closingTime = Date().time / 1000

                    Log.w("Relay", "Relay onFailure $url, ${response?.message} $response")
@@ -161,6 +173,7 @@ class Relay(
        } catch (e: Exception) {
            errorCounter++
            isReady = false
+            afterEOSE = false
            closingTime = Date().time / 1000
            Log.e("Relay", "Relay Invalid $url")
            e.printStackTrace()
@@ -173,6 +186,7 @@ class Relay(
        socket?.close(1000, "Normal close")
        socket = null
        isReady = false
+        afterEOSE = false
    }

    fun sendFilter(requestId: String) {
@@ -186,6 +200,7 @@ class Relay(
                        // println("FILTERSSENT $url $request")
                        socket?.send(request)
                        eventUploadCounterInBytes += request.bytesUsedInMemory()
+                        afterEOSE = false
                    }
                }
            } else {