Improves the async rendering of Base64 content

amethyst/src/main/java/com/vitorpamplona/amethyst/ServiceManager.kt

@@ -30,6 +30,7 @@ import coil.decode.SvgDecoder
 import coil.size.Precision
 import com.vitorpamplona.amethyst.model.Account
 import com.vitorpamplona.amethyst.model.LocalCache
+import com.vitorpamplona.amethyst.service.Base64Fetcher
 import com.vitorpamplona.amethyst.service.NostrAccountDataSource
 import com.vitorpamplona.amethyst.service.NostrChannelDataSource
 import com.vitorpamplona.amethyst.service.NostrChatroomDataSource
@@ -98,6 +99,7 @@ class ServiceManager {
                         add(GifDecoder.Factory())
                     }
                     add(SvgDecoder.Factory())
+                    add(Base64Fetcher.Factory)
                 } // .logger(DebugLogger())
                 .okHttpClient { HttpClientManager.getHttpClient() }
                 .precision(Precision.INEXACT)

amethyst/src/main/java/com/vitorpamplona/amethyst/service/Base64Image.kt

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2024 Vitor Pamplona
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+ * Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+package com.vitorpamplona.amethyst.service
+
+import android.content.Context
+import android.graphics.BitmapFactory
+import android.net.Uri
+import androidx.compose.runtime.Stable
+import androidx.core.graphics.drawable.toDrawable
+import coil.ImageLoader
+import coil.decode.DataSource
+import coil.fetch.DrawableResult
+import coil.fetch.FetchResult
+import coil.fetch.Fetcher
+import coil.request.ImageRequest
+import coil.request.Options
+import com.vitorpamplona.amethyst.commons.richtext.RichTextParser.Companion.base64contentPattern
+import java.util.Base64
+
+@Stable
+class Base64Fetcher(
+    private val options: Options,
+    private val data: Uri,
+) : Fetcher {
+    override suspend fun fetch(): FetchResult {
+        checkNotInMainThread()
+
+        val matcher = base64contentPattern.matcher(data.toString())
+
+        if (matcher.find()) {
+            val base64String = matcher.group(2)
+
+            val byteArray = Base64.getDecoder().decode(base64String)
+            val bitmap = BitmapFactory.decodeByteArray(byteArray, 0, byteArray.size) ?: throw Exception("Unable to load base64 $base64String")
+
+            return DrawableResult(
+                drawable = bitmap.toDrawable(options.context.resources),
+                isSampled = false,
+                dataSource = DataSource.MEMORY,
+            )
+        } else {
+            throw Exception("Unable to load base64 $data")
+        }
+    }
+
+    object Factory : Fetcher.Factory<Uri> {
+        override fun create(
+            data: Uri,
+            options: Options,
+            imageLoader: ImageLoader,
+        ): Fetcher? {
+            return if (base64contentPattern.matcher(data.toString()).find()) {
+                return Base64Fetcher(options, data)
+            } else {
+                null
+            }
+        }
+    }
+}
+
+object Base64Requester {
+    fun imageRequest(
+        context: Context,
+        message: String,
+    ): ImageRequest =
+        ImageRequest
+            .Builder(context)
+            .data(message)
+            .fetcherFactory(Base64Fetcher.Factory)
+            .build()
+}

amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/RichTextViewer.kt

@@ -20,7 +20,6 @@
  */
 package com.vitorpamplona.amethyst.ui.components
 
-import android.util.Base64
 import androidx.compose.foundation.border
 import androidx.compose.foundation.clickable
 import androidx.compose.foundation.layout.Arrangement
@@ -29,7 +28,6 @@ import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.ExperimentalLayoutApi
 import androidx.compose.foundation.layout.FlowRow
 import androidx.compose.foundation.layout.Row
-import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.text.InlineTextContent
 import androidx.compose.foundation.text.appendInlineContent
@@ -49,8 +47,6 @@ import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.graphics.Color
-import androidx.compose.ui.layout.ContentScale
-import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.platform.LocalDensity
 import androidx.compose.ui.platform.LocalFontFamilyResolver
 import androidx.compose.ui.platform.LocalLayoutDirection
@@ -65,10 +61,6 @@ import androidx.compose.ui.unit.LayoutDirection
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.unit.em
 import androidx.lifecycle.viewmodel.compose.viewModel
-import coil.compose.AsyncImagePainter
-import coil.compose.SubcomposeAsyncImage
-import coil.compose.SubcomposeAsyncImageContent
-import coil.request.ImageRequest
 import com.vitorpamplona.amethyst.commons.compose.produceCachedState
 import com.vitorpamplona.amethyst.commons.richtext.Base64Segment
 import com.vitorpamplona.amethyst.commons.richtext.BechSegment
@@ -83,7 +75,6 @@ import com.vitorpamplona.amethyst.commons.richtext.InvoiceSegment
 import com.vitorpamplona.amethyst.commons.richtext.LinkSegment
 import com.vitorpamplona.amethyst.commons.richtext.PhoneSegment
 import com.vitorpamplona.amethyst.commons.richtext.RegularTextSegment
-import com.vitorpamplona.amethyst.commons.richtext.RichTextParser
 import com.vitorpamplona.amethyst.commons.richtext.RichTextViewerState
 import com.vitorpamplona.amethyst.commons.richtext.SchemelessUrlSegment
 import com.vitorpamplona.amethyst.commons.richtext.Segment
@@ -96,7 +87,6 @@ import com.vitorpamplona.amethyst.model.checkForHashtagWithIcon
 import com.vitorpamplona.amethyst.service.CachedRichTextParser
 import com.vitorpamplona.amethyst.ui.actions.CrossfadeIfEnabled
 import com.vitorpamplona.amethyst.ui.components.markdown.RenderContentAsMarkdown
-import com.vitorpamplona.amethyst.ui.note.BlankNote
 import com.vitorpamplona.amethyst.ui.note.LoadUser
 import com.vitorpamplona.amethyst.ui.note.NoteCompose
 import com.vitorpamplona.amethyst.ui.note.toShortenHex
@@ -112,7 +102,6 @@ import fr.acinq.secp256k1.Hex
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.withContext
 
 fun isMarkdown(content: String): Boolean =
     content.startsWith("> ") ||
@@ -445,47 +434,7 @@ private fun RenderWordWithPreview(
         is HashIndexEventSegment -> TagLink(word, true, quotesLeft, backgroundColor, accountViewModel, nav)
         is SchemelessUrlSegment -> NoProtocolUrlRenderer(word)
         is RegularTextSegment -> Text(word.segmentText)
-        is Base64Segment -> ImageFromBase64(word.segmentText)
-    }
-}
-
-@Composable
-fun ImageFromBase64(base64String: String) {
-    val context = LocalContext.current
-    var imageBytes by remember { mutableStateOf<ByteArray?>(null) }
-    LaunchedEffect(base64String) {
-        imageBytes =
-            withContext(Dispatchers.IO) {
-                var base64String2 = base64String.removePrefix("data:image/jpeg;base64,")
-                RichTextParser.imageExtensions.forEach {
-                    base64String2 = base64String2.removePrefix("data:image/$it;base64,")
-                }
-                runCatching { Base64.decode(base64String2, Base64.DEFAULT) }.getOrNull()
-            }
-    }
-
-    if (imageBytes == null) {
-        BlankNote()
-    } else {
-        val request =
-            ImageRequest
-                .Builder(context)
-                .data(imageBytes)
-                .build()
-
-        SubcomposeAsyncImage(
-            model = request,
-            contentDescription = null,
-            contentScale = ContentScale.FillWidth,
-            modifier = Modifier.fillMaxWidth(),
-        ) {
-            when (painter.state) {
-                is AsyncImagePainter.State.Success -> {
-                    SubcomposeAsyncImageContent()
-                }
-                else -> BlankNote()
-            }
-        }
+        is Base64Segment -> ZoomableContentView(word.segmentText, state, accountViewModel)
     }
 }
 
@@ -495,8 +444,10 @@ private fun ZoomableContentView(
     state: RichTextViewerState,
     accountViewModel: AccountViewModel,
 ) {
+    println("AABBCC ZoomableContentView -- ${state.imagesForPager.keys.joinToString("..") { it }}")
     state.imagesForPager[word]?.let {
         Box(modifier = HalfVertPadding) {
+            println("AABBCC ZoomableContentView inside $word")
             ZoomableContentView(it, state.imageList, roundedCorner = true, isFiniteHeight = false, accountViewModel)
         }
     }

amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/RobohashAsyncImage.kt

@@ -20,15 +20,11 @@
  */
 package com.vitorpamplona.amethyst.ui.components
 
-import android.content.Context
-import android.graphics.BitmapFactory
-import android.net.Uri
 import androidx.compose.foundation.Image
 import androidx.compose.material.icons.Icons
 import androidx.compose.material.icons.filled.Face
 import androidx.compose.material3.MaterialTheme
 import androidx.compose.runtime.Composable
-import androidx.compose.runtime.Stable
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.graphics.ColorFilter
@@ -37,22 +33,10 @@ import androidx.compose.ui.graphics.FilterQuality
 import androidx.compose.ui.graphics.drawscope.DrawScope
 import androidx.compose.ui.graphics.vector.rememberVectorPainter
 import androidx.compose.ui.layout.ContentScale
-import androidx.compose.ui.platform.LocalContext
-import androidx.core.graphics.drawable.toDrawable
-import coil.ImageLoader
 import coil.compose.AsyncImage
-import coil.compose.rememberAsyncImagePainter
-import coil.decode.DataSource
-import coil.fetch.DrawableResult
-import coil.fetch.FetchResult
-import coil.fetch.Fetcher
-import coil.request.ImageRequest
-import coil.request.Options
 import com.vitorpamplona.amethyst.commons.robohash.CachedRobohash
-import com.vitorpamplona.amethyst.service.checkNotInMainThread
 import com.vitorpamplona.amethyst.ui.theme.isLight
 import com.vitorpamplona.amethyst.ui.theme.onBackgroundColorFilter
-import java.util.Base64
 
 @Composable
 fun RobohashAsyncImage(
@@ -92,53 +76,34 @@ fun RobohashFallbackAsyncImage(
     loadRobohash: Boolean,
 ) {
     if (model != null && loadProfilePicture) {
-        val isBase64 = model.startsWith("data:image/jpeg;base64,")
-
-        if (isBase64) {
-            val context = LocalContext.current
-            val base64Painter =
-                rememberAsyncImagePainter(
-                    model = Base64Requester.imageRequest(context, model),
+        val painter =
+            if (loadRobohash) {
+                rememberVectorPainter(
+                    image = CachedRobohash.get(robot, MaterialTheme.colorScheme.isLight),
                 )
+            } else {
+                forwardingPainter(
+                    painter =
+                        rememberVectorPainter(
+                            image = Icons.Default.Face,
+                        ),
+                    colorFilter = MaterialTheme.colorScheme.onBackgroundColorFilter,
+                )
+            }
 
-            Image(
-                painter = base64Painter,
-                contentDescription = contentDescription,
-                modifier = modifier,
-                alignment = alignment,
-                contentScale = contentScale,
-                colorFilter = colorFilter,
-            )
-        } else {
-            val painter =
-                if (loadRobohash) {
-                    rememberVectorPainter(
-                        image = CachedRobohash.get(robot, MaterialTheme.colorScheme.isLight),
-                    )
-                } else {
-                    forwardingPainter(
-                        painter =
-                            rememberVectorPainter(
-                                image = Icons.Default.Face,
-                            ),
-                        colorFilter = MaterialTheme.colorScheme.onBackgroundColorFilter,
-                    )
-                }
-
-            AsyncImage(
-                model = model,
-                contentDescription = contentDescription,
-                modifier = modifier,
-                placeholder = painter,
-                fallback = painter,
-                error = painter,
-                alignment = alignment,
-                contentScale = contentScale,
-                alpha = alpha,
-                colorFilter = colorFilter,
-                filterQuality = filterQuality,
-            )
-        }
+        AsyncImage(
+            model = model,
+            contentDescription = contentDescription,
+            modifier = modifier,
+            placeholder = painter,
+            fallback = painter,
+            error = painter,
+            alignment = alignment,
+            contentScale = contentScale,
+            alpha = alpha,
+            colorFilter = colorFilter,
+            filterQuality = filterQuality,
+        )
     } else {
         if (loadRobohash) {
             Image(
@@ -161,48 +126,3 @@ fun RobohashFallbackAsyncImage(
         }
     }
 }
-
-object Base64Requester {
-    fun imageRequest(
-        context: Context,
-        message: String,
-    ): ImageRequest =
-        ImageRequest
-            .Builder(context)
-            .data(message)
-            .fetcherFactory(Base64Fetcher.Factory)
-            .build()
-}
-
-@Stable
-class Base64Fetcher(
-    private val options: Options,
-    private val data: Uri,
-) : Fetcher {
-    override suspend fun fetch(): FetchResult {
-        checkNotInMainThread()
-
-        val base64String = data.toString().removePrefix("data:image/jpeg;base64,")
-
-        val byteArray = Base64.getDecoder().decode(base64String)
-        val bitmap = BitmapFactory.decodeByteArray(byteArray, 0, byteArray.size)
-
-        if (bitmap == null) {
-            throw Exception("Unable to load base64 $base64String")
-        }
-
-        return DrawableResult(
-            drawable = bitmap.toDrawable(options.context.resources),
-            isSampled = false,
-            dataSource = DataSource.MEMORY,
-        )
-    }
-
-    object Factory : Fetcher.Factory<Uri> {
-        override fun create(
-            data: Uri,
-            options: Options,
-            imageLoader: ImageLoader,
-        ): Fetcher = Base64Fetcher(options, data)
-    }
-}

commons/src/main/java/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt

@@ -42,6 +42,47 @@ import java.util.regex.Pattern
 import kotlin.coroutines.cancellation.CancellationException
 
 class RichTextParser {
+    fun createImageContent(
+        fullUrl: String,
+        eventTags: ImmutableListOfLists<String>,
+        description: String?,
+        callbackUri: String? = null,
+    ): MediaUrlImage {
+        val frags = Nip54InlineMetadata().parse(fullUrl)
+        val tags = Nip92MediaAttachments().parse(fullUrl, eventTags.lists)
+
+        return MediaUrlImage(
+            url = fullUrl,
+            description = description ?: frags[FileHeaderEvent.ALT] ?: tags[FileHeaderEvent.ALT],
+            hash = frags[FileHeaderEvent.HASH] ?: tags[FileHeaderEvent.HASH],
+            blurhash = frags[FileHeaderEvent.BLUR_HASH] ?: tags[FileHeaderEvent.BLUR_HASH],
+            dim = frags[FileHeaderEvent.DIMENSION] ?: tags[FileHeaderEvent.DIMENSION],
+            contentWarning = frags["content-warning"] ?: tags["content-warning"],
+            uri = callbackUri,
+            mimeType = frags[FileHeaderEvent.MIME_TYPE] ?: tags[FileHeaderEvent.MIME_TYPE],
+        )
+    }
+
+    fun createVideoContent(
+        fullUrl: String,
+        eventTags: ImmutableListOfLists<String>,
+        description: String?,
+        callbackUri: String? = null,
+    ): MediaUrlVideo {
+        val frags = Nip54InlineMetadata().parse(fullUrl)
+        val tags = Nip92MediaAttachments().parse(fullUrl, eventTags.lists)
+        return MediaUrlVideo(
+            url = fullUrl,
+            description = description ?: frags[FileHeaderEvent.ALT] ?: tags[FileHeaderEvent.ALT],
+            hash = frags[FileHeaderEvent.HASH] ?: tags[FileHeaderEvent.HASH],
+            blurhash = frags[FileHeaderEvent.BLUR_HASH] ?: tags[FileHeaderEvent.BLUR_HASH],
+            dim = frags[FileHeaderEvent.DIMENSION] ?: tags[FileHeaderEvent.DIMENSION],
+            contentWarning = frags["content-warning"] ?: tags["content-warning"],
+            uri = callbackUri,
+            mimeType = frags[FileHeaderEvent.MIME_TYPE] ?: tags[FileHeaderEvent.MIME_TYPE],
+        )
+    }
+
     fun parseMediaUrl(
         fullUrl: String,
         eventTags: ImmutableListOfLists<String>,
@@ -50,50 +91,17 @@ class RichTextParser {
     ): MediaUrlContent? {
         val removedParamsFromUrl = removeQueryParamsForExtensionComparison(fullUrl)
         return if (imageExtensions.any { removedParamsFromUrl.endsWith(it) }) {
-            val frags = Nip54InlineMetadata().parse(fullUrl)
-            val tags = Nip92MediaAttachments().parse(fullUrl, eventTags.lists)
-
-            MediaUrlImage(
-                url = fullUrl,
-                description = description ?: frags[FileHeaderEvent.ALT] ?: tags[FileHeaderEvent.ALT],
-                hash = frags[FileHeaderEvent.HASH] ?: tags[FileHeaderEvent.HASH],
-                blurhash = frags[FileHeaderEvent.BLUR_HASH] ?: tags[FileHeaderEvent.BLUR_HASH],
-                dim = frags[FileHeaderEvent.DIMENSION] ?: tags[FileHeaderEvent.DIMENSION],
-                contentWarning = frags["content-warning"] ?: tags["content-warning"],
-                uri = callbackUri,
-                mimeType = frags[FileHeaderEvent.MIME_TYPE] ?: tags[FileHeaderEvent.MIME_TYPE],
-            )
+            createImageContent(fullUrl, eventTags, description, callbackUri)
         } else if (videoExtensions.any { removedParamsFromUrl.endsWith(it) }) {
-            val frags = Nip54InlineMetadata().parse(fullUrl)
-            val tags = Nip92MediaAttachments().parse(fullUrl, eventTags.lists)
-            MediaUrlVideo(
-                url = fullUrl,
-                description = description ?: frags[FileHeaderEvent.ALT] ?: tags[FileHeaderEvent.ALT],
-                hash = frags[FileHeaderEvent.HASH] ?: tags[FileHeaderEvent.HASH],
-                blurhash = frags[FileHeaderEvent.BLUR_HASH] ?: tags[FileHeaderEvent.BLUR_HASH],
-                dim = frags[FileHeaderEvent.DIMENSION] ?: tags[FileHeaderEvent.DIMENSION],
-                contentWarning = frags["content-warning"] ?: tags["content-warning"],
-                uri = callbackUri,
-                mimeType = frags[FileHeaderEvent.MIME_TYPE] ?: tags[FileHeaderEvent.MIME_TYPE],
-            )
+            createVideoContent(fullUrl, eventTags, description, callbackUri)
         } else {
             null
         }
     }
 
-    private fun parseBase64Images(content: String): LinkedHashSet<String> {
-        val regex = "data:image/(${imageExtensions.joinToString(separator = "|") { it } });base64,[a-zA-Z0-9+/]+={0,2}"
-        val pattern = Pattern.compile(regex)
-        val matcher = pattern.matcher(content)
-
-        val base64Images = mutableListOf<String>()
-
-        // Find all matches and add them to the list
-        while (matcher.find()) {
-            base64Images.add(matcher.group())
-        }
-
-        return base64Images.mapTo(LinkedHashSet(base64Images.size)) { it }
+    private fun checkBase64(content: String): Boolean {
+        val matcher = base64contentPattern.matcher(content)
+        return matcher.find()
     }
 
     fun parseValidUrls(content: String): LinkedHashSet<String> {
@@ -129,16 +137,23 @@ class RichTextParser {
 
         val imagesForPager =
             urlSet.mapNotNull { fullUrl -> parseMediaUrl(fullUrl, tags, content, callbackUri) }.associateBy { it.url }
-        val imageList = imagesForPager.values.toList()
 
         val emojiMap = Nip30CustomEmoji.createEmojiMap(tags)
 
         val segments = findTextSegments(content, imagesForPager.keys, urlSet, emojiMap, tags)
 
+        val base64Images = segments.map { it.words.filterIsInstance<Base64Segment>() }.flatten()
+
+        val imagesForPagerWithBase64 =
+            imagesForPager +
+                base64Images
+                    .map { createImageContent(it.segmentText, tags, content, callbackUri) }
+                    .associateBy { it.url }
+
         return RichTextViewerState(
             urlSet.toImmutableSet(),
-            imagesForPager.toImmutableMap(),
-            imageList.toImmutableList(),
+            imagesForPagerWithBase64.toImmutableMap(),
+            imagesForPagerWithBase64.values.toImmutableList(),
             emojiMap.toImmutableMap(),
             segments,
         )
@@ -218,11 +233,8 @@ class RichTextParser {
     ): Segment {
         if (word.isEmpty()) return RegularTextSegment(word)
 
-        if (word.startsWith("data:image")) {
-            val base64Images = parseBase64Images(word)
-            if (base64Images.isNotEmpty()) {
-                return Base64Segment(word)
-            }
+        if (word.startsWith("data:image/")) {
+            if (checkBase64(word)) return Base64Segment(word)
         }
 
         if (images.contains(word)) return ImageSegment(word)
@@ -338,6 +350,8 @@ class RichTextParser {
         val imageExtensions = listOf("png", "jpg", "gif", "bmp", "jpeg", "webp", "svg", "avif")
         val videoExtensions = listOf("mp4", "avi", "wmv", "mpg", "amv", "webm", "mov", "mp3", "m3u8")
 
+        val base64contentPattern = Pattern.compile("data:image/(${imageExtensions.joinToString(separator = "|") { it } });base64,([a-zA-Z0-9+/]+={0,2})")
+
         val tagIndex = Pattern.compile("\\#\\[([0-9]+)\\](.*)")
         val hashTagsPattern: Pattern =
             Pattern.compile("#([^\\s!@#\$%^&*()=+./,\\[{\\]};:'\"?><]+)(.*)", Pattern.CASE_INSENSITIVE)