From 2ac5742b0e32c1dabe00b352ba36c5939937fa4d Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Fri, 18 Aug 2023 23:42:24 -0400 Subject: [PATCH] Adds a waveform to audio files --- app/build.gradle | 3 + .../ui/components/AudioWaveformReadOnly.kt | 160 ++++++++++++++++++ .../amethyst/ui/components/VideoView.kt | 83 ++++++++- .../amethyst/ui/note/NoteCompose.kt | 2 + .../amethyst/ui/screen/ThreadFeedView.kt | 3 + .../quartz/events/AudioHeaderEvent.kt | 5 +- 6 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 app/src/main/java/com/vitorpamplona/amethyst/ui/components/AudioWaveformReadOnly.kt diff --git a/app/build.gradle b/app/build.gradle index ce30f7b89..22d636935 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -182,6 +182,9 @@ dependencies { // GeoHash implementation 'com.github.drfonfon:android-kotlin-geohash:1.0' + // Waveform visualizer + implementation 'com.github.lincollincol:compose-audiowaveform:1.1.1' + // Video compression lib implementation 'com.github.AbedElazizShe:LightCompressor:1.3.1' // Image compression lib diff --git a/app/src/main/java/com/vitorpamplona/amethyst/ui/components/AudioWaveformReadOnly.kt b/app/src/main/java/com/vitorpamplona/amethyst/ui/components/AudioWaveformReadOnly.kt new file mode 100644 index 000000000..b7f68f3bb --- /dev/null +++ b/app/src/main/java/com/vitorpamplona/amethyst/ui/components/AudioWaveformReadOnly.kt @@ -0,0 +1,160 @@ +package com.vitorpamplona.amethyst.ui.components + +import androidx.compose.animation.core.AnimationSpec +import androidx.compose.animation.core.animateFloatAsState +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.requiredHeight +import androidx.compose.material.MaterialTheme +import androidx.compose.runtime.* +import androidx.compose.ui.ExperimentalComposeUiApi +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.CornerRadius +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.* +import androidx.compose.ui.graphics.drawscope.DrawStyle +import androidx.compose.ui.graphics.drawscope.Fill +import androidx.compose.ui.unit.Dp +import androidx.compose.ui.unit.coerceIn +import androidx.compose.ui.unit.dp +import com.linc.audiowaveform.model.AmplitudeType +import com.linc.audiowaveform.model.WaveformAlignment +import kotlin.math.ceil +import kotlin.math.roundToInt + +private val MinSpikeWidthDp: Dp = 1.dp +private val MaxSpikeWidthDp: Dp = 24.dp +private val MinSpikePaddingDp: Dp = 0.dp +private val MaxSpikePaddingDp: Dp = 12.dp +private val MinSpikeRadiusDp: Dp = 0.dp +private val MaxSpikeRadiusDp: Dp = 12.dp + +private const val MinProgress: Float = 0F +private const val MaxProgress: Float = 1F + +private const val MinSpikeHeight: Float = 1F +private const val DefaultGraphicsLayerAlpha: Float = 0.99F + +@OptIn(ExperimentalComposeUiApi::class) +@Composable +fun AudioWaveformReadOnly( + modifier: Modifier = Modifier, + style: DrawStyle = Fill, + waveformBrush: Brush = SolidColor(Color.White), + progressBrush: Brush = SolidColor(Color.Blue), + waveformAlignment: WaveformAlignment = WaveformAlignment.Center, + amplitudeType: AmplitudeType = AmplitudeType.Avg, + onProgressChangeFinished: (() -> Unit)? = null, + spikeAnimationSpec: AnimationSpec = tween(500), + spikeWidth: Dp = 3.dp, + spikeRadius: Dp = 2.dp, + spikePadding: Dp = 2.dp, + progress: Float = 0F, + amplitudes: List, + onProgressChange: (Float) -> Unit +) { + val backgroundColor = MaterialTheme.colors.background + val _progress = remember(progress) { progress.coerceIn(MinProgress, MaxProgress) } + val _spikeWidth = remember(spikeWidth) { spikeWidth.coerceIn(MinSpikeWidthDp, MaxSpikeWidthDp) } + val _spikePadding = remember(spikePadding) { spikePadding.coerceIn(MinSpikePaddingDp, MaxSpikePaddingDp) } + val _spikeRadius = remember(spikeRadius) { spikeRadius.coerceIn(MinSpikeRadiusDp, MaxSpikeRadiusDp) } + val _spikeTotalWidth = remember(spikeWidth, spikePadding) { _spikeWidth + _spikePadding } + var canvasSize by remember { mutableStateOf(Size(0f, 0f)) } + var spikes by remember { mutableStateOf(0F) } + val spikesAmplitudes = remember(amplitudes, spikes, amplitudeType) { + amplitudes.toDrawableAmplitudes( + amplitudeType = amplitudeType, + spikes = spikes.toInt(), + minHeight = MinSpikeHeight, + maxHeight = canvasSize.height.coerceAtLeast(MinSpikeHeight) + ) + }.map { animateFloatAsState(it, spikeAnimationSpec).value } + Canvas( + modifier = Modifier + .fillMaxWidth() + .requiredHeight(48.dp) + .graphicsLayer(alpha = DefaultGraphicsLayerAlpha) + .then(modifier) + ) { + canvasSize = size + spikes = size.width / _spikeTotalWidth.toPx() + spikesAmplitudes.forEachIndexed { index, amplitude -> + drawRoundRect( + brush = waveformBrush, + topLeft = Offset( + x = index * _spikeTotalWidth.toPx(), + y = when (waveformAlignment) { + WaveformAlignment.Top -> 0F + WaveformAlignment.Bottom -> size.height - amplitude + WaveformAlignment.Center -> size.height / 2F - amplitude / 2F + } + ), + size = Size( + width = _spikeWidth.toPx(), + height = amplitude + ), + cornerRadius = CornerRadius(_spikeRadius.toPx(), _spikeRadius.toPx()), + style = style + ) + drawRect( + brush = progressBrush, + size = Size( + width = _progress * size.width, + height = size.height + ), + blendMode = BlendMode.SrcAtop + ) + } + } +} + +private fun List.toDrawableAmplitudes( + amplitudeType: AmplitudeType, + spikes: Int, + minHeight: Float, + maxHeight: Float +): List { + val amplitudes = map(Int::toFloat) + if (amplitudes.isEmpty() || spikes == 0) { + return List(spikes) { minHeight } + } + val transform = { data: List -> + when (amplitudeType) { + AmplitudeType.Avg -> data.average() + AmplitudeType.Max -> data.max() + AmplitudeType.Min -> data.min() + }.toFloat().coerceIn(minHeight, maxHeight) + } + return when { + spikes > amplitudes.count() -> amplitudes.fillToSize(spikes, transform) + else -> amplitudes.chunkToSize(spikes, transform) + }.normalize(minHeight, maxHeight) +} + +internal fun Iterable.fillToSize(size: Int, transform: (List) -> T): List { + val capacity = ceil(size.safeDiv(count())).roundToInt() + return map { data -> List(capacity) { data } }.flatten().chunkToSize(size, transform) +} + +internal fun Iterable.chunkToSize(size: Int, transform: (List) -> T): List { + val chunkSize = count() / size + val remainder = count() % size + val remainderIndex = ceil(count().safeDiv(remainder)).roundToInt() + val chunkIteration = filterIndexed { index, _ -> + remainderIndex == 0 || index % remainderIndex != 0 + }.chunked(chunkSize, transform) + return when (size) { + chunkIteration.count() -> chunkIteration + else -> chunkIteration.chunkToSize(size, transform) + } +} + +internal fun Iterable.normalize(min: Float, max: Float): List { + return map { (max - min) * ((it - min()) / (max() - min())) + min } +} + +private fun Int.safeDiv(value: Int): Float { + return if (value == 0) return 0F else this / value.toFloat() +} diff --git a/app/src/main/java/com/vitorpamplona/amethyst/ui/components/VideoView.kt b/app/src/main/java/com/vitorpamplona/amethyst/ui/components/VideoView.kt index 73452d896..5de666a67 100644 --- a/app/src/main/java/com/vitorpamplona/amethyst/ui/components/VideoView.kt +++ b/app/src/main/java/com/vitorpamplona/amethyst/ui/components/VideoView.kt @@ -9,6 +9,8 @@ import android.view.ViewGroup import android.widget.FrameLayout import androidx.annotation.OptIn import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.tween import androidx.compose.animation.fadeIn import androidx.compose.animation.fadeOut import androidx.compose.foundation.background @@ -34,6 +36,8 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.composed import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color import androidx.compose.ui.layout.LayoutCoordinates import androidx.compose.ui.layout.boundsInWindow import androidx.compose.ui.layout.onGloballyPositioned @@ -54,6 +58,7 @@ import androidx.media3.ui.AspectRatioFrameLayout import androidx.media3.ui.PlayerView import coil.imageLoader import coil.request.ImageRequest +import com.linc.audiowaveform.infiniteLinearGradient import com.vitorpamplona.amethyst.PlaybackClientController import com.vitorpamplona.amethyst.model.ConnectivityType import com.vitorpamplona.amethyst.service.connectivitystatus.ConnectivityStatus @@ -66,8 +71,11 @@ import com.vitorpamplona.amethyst.ui.theme.PinBottomIconSize import com.vitorpamplona.amethyst.ui.theme.Size22Modifier import com.vitorpamplona.amethyst.ui.theme.Size50Modifier import com.vitorpamplona.amethyst.ui.theme.VolumeBottomIconSize +import kotlinx.collections.immutable.ImmutableList import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.launch import java.util.UUID import kotlin.math.abs @@ -137,6 +145,7 @@ fun VideoView( videoUri: String, title: String? = null, thumb: VideoThumb? = null, + waveform: ImmutableList? = null, artworkUri: String? = null, authorName: String? = null, nostrUriCallback: String? = null, @@ -151,6 +160,7 @@ fun VideoView( defaultToStart, title, thumb, + waveform, artworkUri, authorName, nostrUriCallback, @@ -167,6 +177,7 @@ fun VideoViewInner( defaultToStart: Boolean = false, title: String? = null, thumb: VideoThumb? = null, + waveform: ImmutableList? = null, artworkUri: String? = null, authorName: String? = null, nostrUriCallback: String? = null, @@ -220,7 +231,7 @@ fun VideoViewInner( defaultToStart = defaultToStart, nostrUriCallback = nostrUriCallback ) { controller, keepPlaying -> - RenderVideoPlayer(controller, thumb, keepPlaying, automaticallyStartPlayback, activeOnScreen, onDialog) + RenderVideoPlayer(controller, thumb, waveform, keepPlaying, automaticallyStartPlayback, activeOnScreen, onDialog) } } } @@ -472,6 +483,7 @@ data class VideoThumb( private fun RenderVideoPlayer( controller: MediaController, thumbData: VideoThumb?, + waveform: ImmutableList? = null, keepPlaying: MutableState, automaticallyStartPlayback: MutableState, activeOnScreen: MutableState, @@ -489,7 +501,7 @@ private fun RenderVideoPlayer( AndroidView( modifier = Modifier .fillMaxWidth() - .defaultMinSize(minHeight = 70.dp) + .defaultMinSize(minHeight = 100.dp) .align(Alignment.Center), factory = { PlayerView(context).apply { @@ -518,6 +530,10 @@ private fun RenderVideoPlayer( } ) + waveform?.let { + Waveform(it, controller, Modifier.align(Alignment.Center)) + } + val startingMuteState = remember(controller) { controller.volume < 0.001 } @@ -555,6 +571,69 @@ private fun RenderVideoPlayer( } } +private fun pollCurrentDuration(controller: MediaController) = flow { + while (controller.currentPosition <= controller.duration) { + emit(controller.currentPosition / controller.duration.toFloat()) + delay(100) + } +}.conflate() + +@Composable +fun Waveform( + waveform: ImmutableList, + controller: MediaController, + align: Modifier +) { + val waveformProgress = remember { mutableStateOf(0F) } + + DrawWaveform(waveform, waveformProgress, align) + + val restartFlow = remember { + mutableStateOf(0) + } + + // Keeps the screen on while playing and viewing videos. + DisposableEffect(key1 = controller) { + val listener = object : Player.Listener { + override fun onIsPlayingChanged(isPlaying: Boolean) { + // doesn't consider the mutex because the screen can turn off if the video + // being played in the mutex is not visible. + if (isPlaying) { + restartFlow.value += 1 + } + } + } + + controller.addListener(listener) + onDispose { + controller.removeListener(listener) + } + } + + LaunchedEffect(key1 = restartFlow.value) { + pollCurrentDuration(controller).collect() { value -> + waveformProgress.value = value + } + } +} + +@Composable +fun DrawWaveform(waveform: ImmutableList, waveformProgress: MutableState, align: Modifier) { + AudioWaveformReadOnly( + modifier = align, + amplitudes = waveform, + progress = waveformProgress.value, + progressBrush = Brush.infiniteLinearGradient( + colors = listOf(Color(0xff2598cf), Color(0xff652d80)), + animation = tween(durationMillis = 6000, easing = LinearEasing), + width = 128F + ), + onProgressChange = { + waveformProgress.value = it + } + ) +} + @Composable fun ControlWhenPlayerIsActive( controller: Player, diff --git a/app/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteCompose.kt b/app/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteCompose.kt index 2784cb539..ccbef8894 100644 --- a/app/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteCompose.kt +++ b/app/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteCompose.kt @@ -3386,6 +3386,7 @@ fun AudioTrackHeader(noteEvent: AudioTrackEvent, note: Note, accountViewModel: A @Composable fun AudioHeader(noteEvent: AudioHeaderEvent, note: Note, accountViewModel: AccountViewModel, nav: (String) -> Unit) { val media = remember { noteEvent.stream() ?: noteEvent.download() } + val waveform = remember { noteEvent.wavefrom()?.toImmutableList()?.ifEmpty { null } } val subject = remember { noteEvent.subject()?.ifBlank { null } } val content = remember { noteEvent.content().ifBlank { null } } @@ -3397,6 +3398,7 @@ fun AudioHeader(noteEvent: AudioHeaderEvent, note: Note, accountViewModel: Accou ) { VideoView( videoUri = media, + waveform = waveform, title = noteEvent.subject(), authorName = note.author?.toBestDisplayName(), accountViewModel = accountViewModel, diff --git a/app/src/main/java/com/vitorpamplona/amethyst/ui/screen/ThreadFeedView.kt b/app/src/main/java/com/vitorpamplona/amethyst/ui/screen/ThreadFeedView.kt index ef644c570..2f19115f3 100644 --- a/app/src/main/java/com/vitorpamplona/amethyst/ui/screen/ThreadFeedView.kt +++ b/app/src/main/java/com/vitorpamplona/amethyst/ui/screen/ThreadFeedView.kt @@ -66,6 +66,7 @@ import com.vitorpamplona.amethyst.ui.theme.lessImportantLink import com.vitorpamplona.amethyst.ui.theme.placeholderText import com.vitorpamplona.amethyst.ui.theme.selectedNote import com.vitorpamplona.quartz.events.AppDefinitionEvent +import com.vitorpamplona.quartz.events.AudioHeaderEvent import com.vitorpamplona.quartz.events.AudioTrackEvent import com.vitorpamplona.quartz.events.BadgeDefinitionEvent import com.vitorpamplona.quartz.events.ChannelCreateEvent @@ -383,6 +384,8 @@ fun NoteMaster( DisplayPeopleList(baseNote, backgroundColor, accountViewModel, nav) } else if (noteEvent is AudioTrackEvent) { AudioTrackHeader(noteEvent, baseNote, accountViewModel, nav) + } else if (noteEvent is AudioHeaderEvent) { + AudioHeader(noteEvent, baseNote, accountViewModel, nav) } else if (noteEvent is CommunityPostApprovalEvent) { RenderPostApproval( baseNote, diff --git a/quartz/src/main/java/com/vitorpamplona/quartz/events/AudioHeaderEvent.kt b/quartz/src/main/java/com/vitorpamplona/quartz/events/AudioHeaderEvent.kt index c36533b46..a9ea96d6f 100644 --- a/quartz/src/main/java/com/vitorpamplona/quartz/events/AudioHeaderEvent.kt +++ b/quartz/src/main/java/com/vitorpamplona/quartz/events/AudioHeaderEvent.kt @@ -1,6 +1,7 @@ package com.vitorpamplona.quartz.events import androidx.compose.runtime.Immutable +import com.fasterxml.jackson.module.kotlin.readValue import com.vitorpamplona.quartz.utils.TimeUtils import com.vitorpamplona.quartz.encoders.toHexKey import com.vitorpamplona.quartz.crypto.CryptoUtils @@ -18,7 +19,9 @@ class AudioHeaderEvent( fun download() = tags.firstOrNull { it.size > 1 && it[0] == DOWNLOAD_URL }?.get(1) fun stream() = tags.firstOrNull { it.size > 1 && it[0] == STREAM_URL }?.get(1) - fun wavefrom() = tags.firstOrNull { it.size > 1 && it[0] == WAVEFORM }?.get(1) + fun wavefrom() = tags.firstOrNull { it.size > 1 && it[0] == WAVEFORM }?.get(1)?.let { + mapper.readValue>(it) + } companion object { const val kind = 1808