diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 8e8b259fa9..9dc7eb5d31 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -315,6 +315,7 @@ dependencies { implementation(libs.androidx.media3.ui) implementation(libs.androidx.media3.ui.compose) implementation(libs.ass.media) + implementation(libs.ass.kt) implementation(libs.exoplayerhdrutils) implementation(libs.coil.core) diff --git a/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt b/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt index 5b951288fe..45b5547f18 100644 --- a/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt +++ b/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt @@ -84,6 +84,8 @@ import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Job +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -110,7 +112,9 @@ import org.jellyfin.sdk.api.sockets.subscribe import org.jellyfin.sdk.model.DeviceInfo import org.jellyfin.sdk.model.api.BaseItemKind import org.jellyfin.sdk.model.api.ImageType +import org.jellyfin.sdk.model.api.MediaAttachment import org.jellyfin.sdk.model.api.MediaSegmentType +import org.jellyfin.sdk.model.api.MediaSourceInfo import org.jellyfin.sdk.model.api.MediaStreamType import org.jellyfin.sdk.model.api.MediaType import org.jellyfin.sdk.model.api.PlayMethod @@ -194,6 +198,18 @@ class PlaybackViewModel private val jobs = mutableListOf() private var subscribeJob: Job? = null + /** + * Fonts downloaded for the last played source, reused when only the streams change (audio track, bitrate, etc) + */ + @Volatile + private var assFontCache: AssFontCache? = null + + private class AssFontCache( + val itemId: UUID, + val sourceId: String, + val fonts: List>, + ) + private val isPlaylist = destination is Destination.PlaybackList val subtitleSearchState = MutableStateFlow(SubtitleSearchState()) @@ -624,6 +640,59 @@ class PlaybackViewModel return@withContext true } + /** + * Downloads the fonts attached to [source] if the selected subtitle is ASS/SSA + */ + private suspend fun fetchAssFonts( + itemId: UUID, + source: MediaSourceInfo, + subtitleIndex: Int?, + ): List> { + val sourceId = source.id ?: return emptyList() + val subtitleCodec = + source.mediaStreams + ?.firstOrNull { it.type == MediaStreamType.SUBTITLE && it.index == subtitleIndex } + ?.codec + ?.lowercase() + if (subtitleCodec != Codec.Subtitle.ASS && subtitleCodec != Codec.Subtitle.SSA) + { + return emptyList() + } + + assFontCache + ?.takeIf { it.itemId == itemId && it.sourceId == sourceId } + ?.let { + Timber.d("Reusing %s cached font attachments for ASS subtitles", it.fonts.size) + return it.fonts + } + + val fonts = source.mediaAttachments.orEmpty().filter { it.isFont } + Timber.d("Downloading %s font attachments for ASS subtitles", fonts.size) + val downloaded = + coroutineScope { + fonts + .map { attachment -> + async { + try { + val data = api.videoApi.getAttachment(itemId, sourceId, attachment.index).content + (attachment.fileName ?: "attachment${attachment.index}") to data + } catch (ex: CancellationException) { + throw ex + } catch (ex: Exception) { + Timber.w(ex, "Failed to download font attachment %s", attachment.fileName) + null + } + } + }.awaitAll() + .filterNotNull() + } + // Only cache a complete download so a transient failure is retried on the next stream change + if (downloaded.size == fonts.size) { + assFontCache = AssFontCache(itemId, sourceId, downloaded) + } + return downloaded + } + /** * Change which streams (ie audio or subtitle) are active */ @@ -798,6 +867,21 @@ class PlaybackViewModel } }.build() + // Drop fonts from a previous item/source so they aren't kept in memory for the rest of the session + if (assFontCache?.let { it.itemId != itemId || it.sourceId != source.id } == true) { + assFontCache = null + } + + // libass only finds the fonts in the Matroska attachments, which are missing unless + // the original file is played, so fetch them from the server instead + val assHandler = currentPlayer.value?.assHandler + val assFonts = + if (assHandler != null && transcodeType != PlayMethod.DIRECT_PLAY) { + fetchAssFonts(itemId, source, subtitleIndex) + } else { + emptyList() + } + val playback = CurrentPlayback( item = item, @@ -849,6 +933,12 @@ class PlaybackViewModel mediaItem, positionMs, ) + // Must happen after setMediaItem, since AssHandler drops its buffered fonts on a media item transition. + // libass keeps the fonts that were already added though, so clear them to avoid piling up copies. + if (assHandler != null && assFonts.isNotEmpty()) { + assHandler.ass.clearFont() + assFonts.forEach { (name, data) -> assHandler.addFont(name, data) } + } val onFailure: () -> Unit = { if (player is MpvPlayer && externalSubtitle != null) { // MpvPlayer may change tracks more than once to add external subtitles @@ -1760,3 +1850,33 @@ class PlaybackViewModel } } } + +// Detect fonts like the spec says: https://github.com/ietf-wg-cellar/matroska-specification/blob/1a943ead9bb4c67b02b966cebafe51c8c074e703/cellar-matroska/attachments.md?plain=1#L87-L111 +private val fontMimeTypes = + setOf( + "font/collection", + "font/ttf", + "font/otf", + "font/sfnt", + "font/woff", + "font/woff2", + "application/font-sfnt", + "application/font-woff", + "application/x-truetype-font", + "application/vnd.ms-opentype", + "application/x-font", + "application/x-font-ttf", + "application/x-font-otf", + ) + +private val fontExtensions = listOf(".ttf", ".otf", ".ttc", ".otc") + +private val MediaAttachment.isFont: Boolean + get() { + val mime = mimeType?.lowercase() ?: return false + return mime in fontMimeTypes || + ( + mime == "application/octet-stream" && + fileName?.lowercase()?.let { name -> fontExtensions.any { name.endsWith(it) } } == true + ) + } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 793f979dcd..1c8c63d005 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -152,6 +152,7 @@ androidx-core-testing = { module = "androidx.arch.core:core-testing", version.re androidx-runner = { group = "androidx.test", name = "runner", version.ref = "runner" } ass-media = { group = "io.github.peerless2012", name = "ass-media", version.ref = "libass-android" } +ass-kt = { group = "io.github.peerless2012", name = "ass-kt", version.ref = "libass-android" } junit = { group = "junit", name = "junit", version.ref = "junit" } androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } androidx-espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version.ref = "espressoCore" }