highperfocused/amethyst
1
0
Fork 0
mirror of https://github.com/vitorpamplona/amethyst.git synced 2025-03-26 17:52:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Speeding up the loop through local cache.

Browse Source
This commit is contained in:
Vitor Pamplona 2024-02-26 20:51:59 -05:00
parent 8ebad524ff
commit 11ff0736e4
22 changed files with 122 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/src/main/java/com/vitorpamplona/amethyst/model/Account.kt
View File

@ -1202,7 +1202,7 @@ class Account(
Client.send(signedEvent, relayList = relayList)
LocalCache.consume(signedEvent, null)
return LocalCache.notes[signedEvent.id]
return LocalCache.getNoteIfExists(signedEvent.id)
}
fun consumeNip95(
@ -1212,7 +1212,7 @@ class Account(
LocalCache.consume(data, null)
LocalCache.consume(signedEvent, null)
return LocalCache.notes[signedEvent.id]
return LocalCache.getNoteIfExists(signedEvent.id)
}
fun sendNip95(
@ -1232,7 +1232,7 @@ class Account(
Client.send(signedEvent, relayList = relayList)
LocalCache.consume(signedEvent, null)
LocalCache.notes[signedEvent.id]?.let { onReady(it) }
LocalCache.getNoteIfExists(signedEvent.id)?.let { onReady(it) }
}
fun createHeader(

112
app/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt
View File

@ -102,6 +102,7 @@ import com.vitorpamplona.quartz.events.TextNoteEvent
import com.vitorpamplona.quartz.events.VideoHorizontalEvent
import com.vitorpamplona.quartz.events.VideoVerticalEvent
import com.vitorpamplona.quartz.events.WikiNoteEvent
import com.vitorpamplona.quartz.utils.TimeUtils
import kotlinx.collections.immutable.ImmutableList
import kotlinx.collections.immutable.persistentSetOf
import kotlinx.collections.immutable.toImmutableList
@ -118,18 +119,31 @@ import java.time.Instant
import java.time.ZoneId
import java.time.format.DateTimeFormatter
import java.util.concurrent.ConcurrentHashMap
import kotlin.time.measureTimedValue
object LocalCache {
val antiSpam = AntiSpamFilter()
val users = ConcurrentHashMap<HexKey, User>(5000)
val notes = ConcurrentHashMap<HexKey, Note>(5000)
private val users = ConcurrentHashMap<HexKey, User>(5000)
private val notes = ConcurrentHashMap<HexKey, Note>(5000)
val channels = ConcurrentHashMap<HexKey, Channel>()
val addressables = ConcurrentHashMap<String, AddressableNote>(100)
val awaitingPaymentRequests =
ConcurrentHashMap<HexKey, Pair<Note?, (LnZapPaymentResponseEvent) -> Unit>>(10)
var noteListCache: List<Note> = emptyList()
var userListCache: List<User> = emptyList()
fun updateListCache() {
val (value, elapsed) =
measureTimedValue {
noteListCache = ArrayList(notes.values)
userListCache = ArrayList(users.values)
}
Log.d("LocalCache", "UpdateListCache $elapsed")
}
fun checkGetOrCreateUser(key: String): User? {
// checkNotInMainThread()
@ -910,7 +924,7 @@ object LocalCache {
event
.deleteEvents()
.mapNotNull { notes[it] }
.mapNotNull { getNoteIfExists(it) }
.forEach { deleteNote ->
// must be the same author
if (deleteNote.author?.pubkeyHex == event.pubKey) {
@ -1547,11 +1561,14 @@ object LocalCache {
val key = decodePublicKeyAsHexOrNull(username)
if (key != null && users[key] != null) {
return listOfNotNull(users[key])
if (key != null) {
val user = getUserIfExists(key)
if (user != null) {
return listOfNotNull(user)
}
}
return users.values.filter {
return userListCache.filter {
(it.anyNameStartsWith(username)) ||
it.pubkeyHex.startsWith(username, true) ||
it.pubkeyNpub().startsWith(username, true)
@ -1569,11 +1586,14 @@ object LocalCache {
null
}
if (key != null && (notes[key] ?: addressables[key]) != null) {
return listOfNotNull(notes[key] ?: addressables[key])
if (key != null) {
val note = getNoteIfExists(key)
if (note != null) {
return listOfNotNull(note)
}
}
return notes.values.filter {
return noteListCache.filter {
(
it.event !is GenericRepostEvent &&
it.event !is RepostEvent &&
@ -1652,28 +1672,29 @@ object LocalCache {
suspend fun findEarliestOtsForNote(note: Note): Long? {
checkNotInMainThread()
val validOts =
notes
.mapNotNull {
val noteEvent = it.value.event
if ((noteEvent is OtsEvent && noteEvent.isTaggedEvent(note.idHex) && !noteEvent.isExpired())) {
noteEvent.verifiedTime
} else {
null
var minTime: Long? = null
val time = TimeUtils.now()
noteListCache.forEach { item ->
val noteEvent = item.event
if ((noteEvent is OtsEvent && noteEvent.isTaggedEvent(note.idHex) && !noteEvent.isExpirationBefore(time))) {
noteEvent.verifiedTime?.let { stampedTime ->
if (minTime == null || stampedTime < (minTime ?: Long.MAX_VALUE)) {
minTime = stampedTime
}
}
}
}
if (validOts.isEmpty()) return null
return validOts.minBy { it }
return minTime
}
fun cleanObservers() {
notes.forEach { it.value.clearLive() }
noteListCache.forEach { it.clearLive() }
addressables.forEach { it.value.clearLive() }
users.forEach { it.value.clearLive() }
userListCache.forEach { it.clearLive() }
}
fun pruneOldAndHiddenMessages(account: Account) {
@ -1699,8 +1720,8 @@ object LocalCache {
}
}
users.forEach { userPair ->
userPair.value.privateChatrooms.values.map {
userListCache.forEach { userPair ->
userPair.privateChatrooms.values.map {
val toBeRemoved = it.pruneMessagesToTheLatestOnly()
val childrenToBeRemoved = mutableListOf<Note>()
@ -1715,7 +1736,7 @@ object LocalCache {
if (toBeRemoved.size > 1) {
println(
"PRUNE: ${toBeRemoved.size} private messages with ${userPair.value.toBestDisplayName()} removed. ${it.roomMessages.size} kept",
"PRUNE: ${toBeRemoved.size} private messages with ${userPair.toBestDisplayName()} removed. ${it.roomMessages.size} kept",
)
}
}
@ -1724,9 +1745,9 @@ object LocalCache {
fun prunePastVersionsOfReplaceables() {
val toBeRemoved =
notes
noteListCache
.filter {
val noteEvent = it.value.event
val noteEvent = it.event
if (noteEvent is AddressableEvent) {
noteEvent.createdAt() <
(addressables[noteEvent.address().toTag()]?.event?.createdAt() ?: 0)
@ -1734,7 +1755,6 @@ object LocalCache {
false
}
}
.values
val childrenToBeRemoved = mutableListOf<Note>()
@ -1759,24 +1779,23 @@ object LocalCache {
checkNotInMainThread()
val toBeRemoved =
notes
noteListCache
.filter {
(
(it.value.event is TextNoteEvent && !it.value.isNewThread()) ||
it.value.event is ReactionEvent ||
it.value.event is LnZapEvent ||
it.value.event is LnZapRequestEvent ||
it.value.event is ReportEvent ||
it.value.event is GenericRepostEvent
(it.event is TextNoteEvent && !it.isNewThread()) ||
it.event is ReactionEvent ||
it.event is LnZapEvent ||
it.event is LnZapRequestEvent ||
it.event is ReportEvent ||
it.event is GenericRepostEvent
) &&
it.value.replyTo?.any { it.liveSet?.isInUse() == true } != true &&
it.value.liveSet?.isInUse() != true && // don't delete if observing.
it.value.author?.pubkeyHex !in
it.replyTo?.any { it.liveSet?.isInUse() == true } != true &&
it.liveSet?.isInUse() != true && // don't delete if observing.
it.author?.pubkeyHex !in
accounts && // don't delete if it is the logged in account
it.value.event?.isTaggedUsers(accounts) !=
it.event?.isTaggedUsers(accounts) !=
true // don't delete if it's a notification to the logged in user
}
.values
val childrenToBeRemoved = mutableListOf<Note>()
@ -1842,7 +1861,8 @@ object LocalCache {
fun pruneExpiredEvents() {
checkNotInMainThread()
val toBeRemoved = notes.filter { it.value.event?.isExpired() == true }.values
val now = TimeUtils.now()
val toBeRemoved = noteListCache.filter { it.event?.isExpirationBefore(now) == true }
val childrenToBeRemoved = mutableListOf<Note>()
@ -1868,7 +1888,7 @@ object LocalCache {
?.hiddenUsers
?.map { userHex ->
(
notes.values.filter { it.event?.pubKey() == userHex } +
noteListCache.filter { it.event?.pubKey() == userHex } +
addressables.values.filter { it.event?.pubKey() == userHex }
)
.toSet()
@ -1890,7 +1910,7 @@ object LocalCache {
checkNotInMainThread()
var removingContactList = 0
users.values.forEach {
userListCache.forEach {
if (
it.pubkeyHex !in loggedIn &&
(it.liveSet == null || it.liveSet?.isInUse() == false) &&
@ -2044,6 +2064,10 @@ class LocalCacheLiveData {
private val bundler = BundledInsert<Note>(1000, Dispatchers.IO)
fun invalidateData(newNote: Note) {
bundler.invalidateList(newNote) { bundledNewNotes -> _newEventBundles.emit(bundledNewNotes) }
bundler.invalidateList(newNote) {
bundledNewNotes ->
_newEventBundles.emit(bundledNewNotes)
LocalCache.updateListCache()
}
}
}

8
app/src/main/java/com/vitorpamplona/amethyst/model/User.kt
View File

@ -139,10 +139,10 @@ class User(val pubkeyHex: String) {
// Update Followers of the past user list
// Update Followers of the new contact list
(oldContactListEvent)?.unverifiedFollowKeySet()?.forEach {
LocalCache.users[it]?.liveSet?.innerFollowers?.invalidateData()
LocalCache.getUserIfExists(it)?.liveSet?.innerFollowers?.invalidateData()
}
(latestContactList)?.unverifiedFollowKeySet()?.forEach {
LocalCache.users[it]?.liveSet?.innerFollowers?.invalidateData()
LocalCache.getUserIfExists(it)?.liveSet?.innerFollowers?.invalidateData()
}
liveSet?.innerRelays?.invalidateData()
@ -363,7 +363,7 @@ class User(val pubkeyHex: String) {
}
suspend fun transientFollowerCount(): Int {
return LocalCache.users.count { it.value.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
return LocalCache.userListCache.count { it.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
}
fun cachedFollowingKeySet(): Set<HexKey> {
@ -387,7 +387,7 @@ class User(val pubkeyHex: String) {
}
suspend fun cachedFollowerCount(): Int {
return LocalCache.users.count { it.value.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
return LocalCache.userListCache.count { it.latestContactList?.isTaggedUser(pubkeyHex) ?: false }
}
fun hasSentMessagesTo(key: ChatroomKey?): Boolean {

6
app/src/main/java/com/vitorpamplona/amethyst/service/notifications/EventNotificationConsumer.kt
View File

@ -112,7 +112,7 @@ class EventNotificationConsumer(private val applicationContext: Context) {
event.pubKey != acc.userProfile().pubkeyHex
) { // from the user
val chatNote = LocalCache.notes[event.id] ?: return
val chatNote = LocalCache.getNoteIfExists(event.id) ?: return
val chatRoom = event.chatroomKey(acc.keyPair.pubKey.toHexKey())
val followingKeySet = acc.followingKeySet()
@ -145,7 +145,7 @@ class EventNotificationConsumer(private val applicationContext: Context) {
event: PrivateDmEvent,
acc: Account,
) {
val note = LocalCache.notes[event.id] ?: return
val note = LocalCache.getNoteIfExists(event.id) ?: return
// old event being re-broadcast
if (event.createdAt < TimeUtils.fiveMinutesAgo()) return
@ -184,7 +184,7 @@ class EventNotificationConsumer(private val applicationContext: Context) {
event: LnZapEvent,
acc: Account,
) {
val noteZapEvent = LocalCache.notes[event.id] ?: return
val noteZapEvent = LocalCache.getNoteIfExists(event.id) ?: return
// old event being re-broadcast
if (event.createdAt < TimeUtils.fiveMinutesAgo()) return

8
app/src/main/java/com/vitorpamplona/amethyst/ui/components/MarkdownParser.kt
View File

@ -66,19 +66,19 @@ class MarkdownParser {
private fun getDisplayNameFromNip19(nip19: Nip19Bech32.Return): Pair<String, String>? {
if (nip19.type == Nip19Bech32.Type.USER) {
LocalCache.users[nip19.hex]?.let {
LocalCache.getUserIfExists(nip19.hex)?.let {
return Pair(it.toBestDisplayName(), it.pubkeyNpub())
}
} else if (nip19.type == Nip19Bech32.Type.NOTE) {
LocalCache.notes[nip19.hex]?.let {
LocalCache.getNoteIfExists(nip19.hex)?.let {
return Pair(it.idDisplayNote(), it.toNEvent())
}
} else if (nip19.type == Nip19Bech32.Type.ADDRESS) {
LocalCache.addressables[nip19.hex]?.let {
LocalCache.getAddressableNoteIfExists(nip19.hex)?.let {
return Pair(it.idDisplayNote(), it.toNEvent())
}
} else if (nip19.type == Nip19Bech32.Type.EVENT) {
LocalCache.notes[nip19.hex]?.let {
LocalCache.getNoteIfExists(nip19.hex)?.let {
return Pair(it.idDisplayNote(), it.toNEvent())
}
}

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/CommunityFeedFilter.kt
View File

@ -33,7 +33,7 @@ class CommunityFeedFilter(val note: AddressableNote, val account: Account) :
}
override fun feed(): List<Note> {
return sort(innerApplyFilter(LocalCache.notes.values))
return sort(innerApplyFilter(LocalCache.noteListCache))
}
override fun applyFilter(collection: Set<Note>): Set<Note> {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/GeoHashFeedFilter.kt
View File

@ -36,7 +36,7 @@ class GeoHashFeedFilter(val tag: String, val account: Account) : AdditiveFeedFil
}
override fun feed(): List<Note> {
return sort(innerApplyFilter(LocalCache.notes.values))
return sort(innerApplyFilter(LocalCache.noteListCache))
}
override fun applyFilter(collection: Set<Note>): Set<Note> {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/HashtagFeedFilter.kt
View File

@ -36,7 +36,7 @@ class HashtagFeedFilter(val tag: String, val account: Account) : AdditiveFeedFil
}
override fun feed(): List<Note> {
return sort(innerApplyFilter(LocalCache.notes.values))
return sort(innerApplyFilter(LocalCache.noteListCache))
}
override fun applyFilter(collection: Set<Note>): Set<Note> {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/HomeConversationsFeedFilter.kt
View File

@ -45,7 +45,7 @@ class HomeConversationsFeedFilter(val account: Account) : AdditiveFeedFilter<Not
}
override fun feed(): List<Note> {
return sort(innerApplyFilter(LocalCache.notes.values))
return sort(innerApplyFilter(LocalCache.noteListCache))
}
override fun applyFilter(collection: Set<Note>): Set<Note> {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/HomeNewThreadFeedFilter.kt
View File

@ -50,7 +50,7 @@ class HomeNewThreadFeedFilter(val account: Account) : AdditiveFeedFilter<Note>()
}
override fun feed(): List<Note> {
val notes = innerApplyFilter(LocalCache.notes.values, true)
val notes = innerApplyFilter(LocalCache.noteListCache, true)
val longFormNotes = innerApplyFilter(LocalCache.addressables.values, false)
return sort(notes + longFormNotes)

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/NotificationFeedFilter.kt
View File

@ -52,7 +52,7 @@ class NotificationFeedFilter(val account: Account) : AdditiveFeedFilter<Note>()
}
override fun feed(): List<Note> {
return sort(innerApplyFilter(LocalCache.notes.values))
return sort(innerApplyFilter(LocalCache.noteListCache))
}
override fun applyFilter(collection: Set<Note>): Set<Note> {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileConversationsFeedFilter.kt
View File

@ -36,7 +36,7 @@ class UserProfileConversationsFeedFilter(val user: User, val account: Account) :
}
override fun feed(): List<Note> {
return sort(innerApplyFilter(LocalCache.notes.values))
return sort(innerApplyFilter(LocalCache.noteListCache))
}
override fun applyFilter(collection: Set<Note>): Set<Note> {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileFollowersFeedFilter.kt
View File

@ -30,7 +30,7 @@ class UserProfileFollowersFeedFilter(val user: User, val account: Account) : Fee
}
override fun feed(): List<User> {
return LocalCache.users.values.filter { it.isFollowing(user) && !account.isHidden(it) }
return LocalCache.userListCache.filter { it.isFollowing(user) && !account.isHidden(it) }
}
override fun limit() = 400

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/UserProfileNewThreadFeedFilter.kt
View File

@ -41,7 +41,7 @@ class UserProfileNewThreadFeedFilter(val user: User, val account: Account) :
}
override fun feed(): List<Note> {
val notes = innerApplyFilter(LocalCache.notes.values)
val notes = innerApplyFilter(LocalCache.noteListCache)
val longFormNotes = innerApplyFilter(LocalCache.addressables.values)
return sort(notes + longFormNotes)

2
app/src/main/java/com/vitorpamplona/amethyst/ui/dal/VideoFeedFilter.kt
View File

@ -43,7 +43,7 @@ class VideoFeedFilter(val account: Account) : AdditiveFeedFilter<Note>() {
}
override fun feed(): List<Note> {
val notes = innerApplyFilter(LocalCache.notes.values)
val notes = innerApplyFilter(LocalCache.noteListCache)
return sort(notes)
}

16
app/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppTopBar.kt
View File

@ -979,11 +979,11 @@ fun debugState(context: Context) {
Log.d(
"STATE DUMP",
"Notes: " +
LocalCache.notes.filter { it.value.liveSet != null }.size +
LocalCache.noteListCache.filter { it.liveSet != null }.size +
" / " +
LocalCache.notes.filter { it.value.event != null }.size +
LocalCache.noteListCache.filter { it.event != null }.size +
" / " +
LocalCache.notes.size,
LocalCache.noteListCache.size,
)
Log.d(
"STATE DUMP",
@ -997,21 +997,21 @@ fun debugState(context: Context) {
Log.d(
"STATE DUMP",
"Users: " +
LocalCache.users.filter { it.value.liveSet != null }.size +
LocalCache.userListCache.filter { it.liveSet != null }.size +
" / " +
LocalCache.users.filter { it.value.info?.latestMetadata != null }.size +
LocalCache.userListCache.filter { it.info?.latestMetadata != null }.size +
" / " +
LocalCache.users.size,
LocalCache.userListCache.size,
)
Log.d(
"STATE DUMP",
"Memory used by Events: " +
LocalCache.notes.values.sumOf { it.event?.countMemory() ?: 0 } / (1024 * 1024) +
LocalCache.noteListCache.sumOf { it.event?.countMemory() ?: 0 } / (1024 * 1024) +
" MB",
)
LocalCache.notes.values
LocalCache.noteListCache
.groupBy { it.event?.kind() }
.forEach { Log.d("STATE DUMP", "Kind ${it.key}: \t${it.value.size} elements ") }
LocalCache.addressables.values

2
app/src/main/java/com/vitorpamplona/amethyst/ui/note/UserReactionsRow.kt
View File

@ -232,7 +232,7 @@ class UserReactionsViewModel(val account: Account) : ViewModel() {
val replies = mutableMapOf<String, Int>()
val takenIntoAccount = mutableSetOf<HexKey>()
LocalCache.notes.values.forEach {
LocalCache.noteListCache.forEach {
val noteEvent = it.event
if (noteEvent != null && !takenIntoAccount.contains(noteEvent.id())) {
if (noteEvent is ReactionEvent) {

2
app/src/main/java/com/vitorpamplona/amethyst/ui/screen/CardFeedViewModel.kt
View File

@ -179,7 +179,7 @@ open class CardFeedViewModel(val localFilter: FeedFilter<Note>) : ViewModel() {
val event = (zapEvent.event as LnZapEvent)
val author =
event.zappedAuthor().firstNotNullOfOrNull {
LocalCache.users[it] // don't create user if it doesn't exist
LocalCache.getUserIfExists(it) // don't create user if it doesn't exist
}
if (author != null) {
val zapRequest = author.zaps.filter { it.value == zapEvent }.keys.firstOrNull()

13
app/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt
View File

@ -96,6 +96,7 @@ import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
import java.util.Locale
import kotlin.coroutines.resume
@ -897,18 +898,22 @@ class AccountViewModel(val account: Account, val settings: SettingsState) : View
return LocalCache.addressables[key]
}
fun findStatusesForUser(
suspend fun findStatusesForUser(
myUser: User,
onResult: (ImmutableList<AddressableNote>) -> Unit,
) {
viewModelScope.launch(Dispatchers.IO) { onResult(LocalCache.findStatusesForUser(myUser)) }
withContext(Dispatchers.IO) {
onResult(LocalCache.findStatusesForUser(myUser))
}
}
fun findOtsEventsForNote(
suspend fun findOtsEventsForNote(
note: Note,
onResult: (Long?) -> Unit,
) {
viewModelScope.launch(Dispatchers.IO) { onResult(LocalCache.findEarliestOtsForNote(note)) }
withContext(Dispatchers.IO) {
onResult(LocalCache.findEarliestOtsForNote(note))
}
}
private suspend fun checkGetOrCreateChannel(key: HexKey): Channel? {

11
quartz/src/main/java/com/vitorpamplona/quartz/encoders/Lud06.kt
View File

@ -33,11 +33,14 @@ class Lud06 {
val url = toLnUrlp(str)
val matcher = LNURLP_PATTERN.matcher(url)
matcher.find()
val domain = matcher.group(2)
val username = matcher.group(3)
if (matcher.find()) {
val domain = matcher.group(2)
val username = matcher.group(3)
"$username@$domain"
"$username@$domain"
} else {
null
}
} catch (t: Throwable) {
t.printStackTrace()
Log.w("Lud06ToLud16", "Fail to convert LUD06 to LUD16", t)

2
quartz/src/main/java/com/vitorpamplona/quartz/events/Event.kt
View File

@ -207,6 +207,8 @@ open class Event(
override fun isExpired() = (expiration() ?: Long.MAX_VALUE) < TimeUtils.now()
override fun isExpirationBefore(time: Long) = (expiration() ?: Long.MAX_VALUE) < time
override fun getTagOfAddressableKind(kind: Int): ATag? {
val kindStr = kind.toString()
val aTag =

2
quartz/src/main/java/com/vitorpamplona/quartz/events/EventInterface.kt
View File

@ -137,5 +137,7 @@ interface EventInterface {
fun isExpired(): Boolean
fun isExpirationBefore(time: Long): Boolean
fun hasZapSplitSetup(): Boolean
}