From a65fb70a4e157d519e0c5915485083fbb0095da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaquim=20St=C3=A4hli?= Date: Thu, 31 Oct 2024 16:10:18 +0100 Subject: [PATCH] SpriteSheet as Image track --- .../core/business/source/SRGAssetLoader.kt | 7 +- .../business/source/SpriteSheetMediaPeriod.kt | 184 ++++++++++++++++++ .../business/source/SpriteSheetMediaSource.kt | 56 ++++++ 3 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt create mode 100644 pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt index 52ebc1ed7..f7dc3adad 100644 --- a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SRGAssetLoader.kt @@ -12,6 +12,7 @@ import androidx.media3.common.MediaMetadata import androidx.media3.common.MimeTypes import androidx.media3.datasource.DataSource.Factory import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.MergingMediaSource import ch.srgssr.pillarbox.core.business.HttpResultException import ch.srgssr.pillarbox.core.business.akamai.AkamaiTokenDataSource import ch.srgssr.pillarbox.core.business.akamai.AkamaiTokenProvider @@ -135,8 +136,12 @@ class SRGAssetLoader internal constructor( .setDrmConfiguration(fillDrmConfiguration(resource)) .setUri(uri) .build() + val contentMediaSource = mediaSourceFactory.createMediaSource(loadingMediaItem) + val mediaSource = chapter.spriteSheet?.let { + MergingMediaSource(contentMediaSource, SpriteSheetMediaSource(it)) + } ?: contentMediaSource return Asset( - mediaSource = mediaSourceFactory.createMediaSource(loadingMediaItem), + mediaSource = mediaSource, trackersData = trackerData.toMediaItemTrackerData(), mediaMetadata = mediaItem.mediaMetadata.buildUpon().apply { defaultMediaMetadata.invoke(this, mediaItem.mediaMetadata, chapter, result) diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt new file mode 100644 index 000000000..fdbf23de6 --- /dev/null +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaPeriod.kt @@ -0,0 +1,184 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.core.business.source + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes +import androidx.media3.common.TrackGroup +import androidx.media3.decoder.DecoderInputBuffer +import androidx.media3.exoplayer.FormatHolder +import androidx.media3.exoplayer.LoadingInfo +import androidx.media3.exoplayer.SeekParameters +import androidx.media3.exoplayer.source.MediaPeriod +import androidx.media3.exoplayer.source.SampleStream +import androidx.media3.exoplayer.source.TrackGroupArray +import androidx.media3.exoplayer.trackselection.ExoTrackSelection +import ch.srgssr.pillarbox.core.business.integrationlayer.data.SpriteSheet +import java.io.ByteArrayOutputStream +import java.net.URL +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.math.max +import kotlin.time.Duration.Companion.milliseconds + +/** + * Sprite sheet media period + */ +internal class SpriteSheetMediaPeriod(private val spriteSheet: SpriteSheet) : MediaPeriod { + private lateinit var bitmap: Bitmap + private val isLoading = AtomicBoolean(true) + private val format = spriteSheet.let { + Format.Builder() + .setId(it.urn) + .setFrameRate(1f / it.interval.milliseconds.inWholeSeconds) + .setCustomData(it) + .setRoleFlags(C.ROLE_FLAG_MAIN) + .setContainerMimeType(MimeTypes.IMAGE_JPEG) + .setSampleMimeType(MimeTypes.IMAGE_JPEG) + .build() + } + private val tracks = TrackGroupArray((TrackGroup("sprite-sheet-srg", format))) + private var positionUs = 0L + + override fun prepare(callback: MediaPeriod.Callback, positionUs: Long) { + this.positionUs = positionUs + callback.onPrepared(this) + URL(spriteSheet.url).openStream().use { + isLoading.set(true) + bitmap = BitmapFactory.decodeStream(it) + isLoading.set(false) + } + } + + fun releasePeriod() { + bitmap.recycle() + } + + override fun selectTracks( + selections: Array, + mayRetainStreamFlags: BooleanArray, + streams: Array, + streamResetFlags: BooleanArray, + positionUs: Long + ): Long { + this.positionUs = positionUs + for (i in selections.indices) { + if (streams[i] != null && (selections[i] == null || !mayRetainStreamFlags[i])) { + streams[i] = null + } + if (streams[i] == null && selections[i] != null) { + val stream = SpriteSheetSampleStream() + streams[i] = stream + streamResetFlags[i] = true + } + } + return positionUs + } + + override fun getTrackGroups(): TrackGroupArray { + return tracks + } + + override fun getBufferedPositionUs(): Long { + return C.TIME_END_OF_SOURCE + } + + override fun getNextLoadPositionUs(): Long { + return C.TIME_END_OF_SOURCE + } + + override fun continueLoading(loadingInfo: LoadingInfo): Boolean { + return isLoading.get() + } + + override fun isLoading(): Boolean { + return isLoading.get() + } + + override fun reevaluateBuffer(positionUs: Long) = Unit + + override fun maybeThrowPrepareError() = Unit + + override fun discardBuffer(positionUs: Long, toKeyframe: Boolean) = Unit + + override fun readDiscontinuity(): Long { + return C.TIME_UNSET + } + + override fun seekToUs(positionUs: Long): Long { + this.positionUs = positionUs + return positionUs + } + + override fun getAdjustedSeekPositionUs(positionUs: Long, seekParameters: SeekParameters): Long { + val intervalUs = spriteSheet.interval.milliseconds.inWholeMicroseconds + return (positionUs / intervalUs) * intervalUs + } + + internal inner class SpriteSheetSampleStream : SampleStream { + private var streamState = STREAM_STATE_SEND_FORMAT + + override fun isReady(): Boolean { + return !isLoading() + } + + override fun maybeThrowError() = Unit + + @Suppress("ReturnCount") + override fun readData(formatHolder: FormatHolder, buffer: DecoderInputBuffer, readFlags: Int): Int { + if (streamState == STREAM_STATE_END_OF_STREAM) { + buffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM) + return C.RESULT_BUFFER_READ + } + + if ((readFlags and SampleStream.FLAG_REQUIRE_FORMAT) != 0 || streamState == STREAM_STATE_SEND_FORMAT) { + formatHolder.format = tracks[0].getFormat(0) + streamState = STREAM_STATE_SEND_SAMPLE + return C.RESULT_FORMAT_READ + } + val intervalUs = spriteSheet.interval.milliseconds.inWholeMicroseconds + val tileIndex = positionUs / intervalUs + buffer.addFlag(C.BUFFER_FLAG_KEY_FRAME) + buffer.timeUs = positionUs + if ((readFlags and SampleStream.FLAG_OMIT_SAMPLE_DATA) == 0) { + val data = cropTileFromImageGrid(max((tileIndex.toInt() - 1), 0)) + buffer.ensureSpaceForWrite(data.size) + buffer.data!!.put(data, /* offset= */0, data.size) + } + if ((readFlags and SampleStream.FLAG_PEEK) == 0 && tileIndex >= (spriteSheet.rows * spriteSheet.columns) - 1) { + streamState = STREAM_STATE_END_OF_STREAM + } + return C.RESULT_BUFFER_READ + } + + override fun skipData(positionUs: Long): Int { + return 0 + } + + private fun cropTileFromImageGrid(tileIndex: Int): ByteArray { + val tileWidth: Int = spriteSheet.thumbnailWidth + val tileHeight: Int = spriteSheet.thumbnailHeight + val tileStartXCoordinate: Int = tileWidth * (tileIndex % spriteSheet.columns) + val tileStartYCoordinate: Int = tileHeight * (tileIndex / spriteSheet.columns) + val tile = Bitmap.createBitmap(bitmap, tileStartXCoordinate, tileStartYCoordinate, tileWidth, tileHeight) + return bitmapToByteArray(tile, Bitmap.CompressFormat.JPEG, MAX_QUALITY) + } + } + + private companion object { + private const val STREAM_STATE_SEND_FORMAT: Int = 0 + private const val STREAM_STATE_SEND_SAMPLE: Int = 1 + private const val STREAM_STATE_END_OF_STREAM: Int = 2 + private const val MAX_QUALITY = 100 + + fun bitmapToByteArray(bitmap: Bitmap, format: Bitmap.CompressFormat, quality: Int): ByteArray { + val stream = ByteArrayOutputStream() + bitmap.compress(format, quality, stream) + return stream.toByteArray() + } + } +} diff --git a/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt new file mode 100644 index 000000000..b0155914e --- /dev/null +++ b/pillarbox-core-business/src/main/java/ch/srgssr/pillarbox/core/business/source/SpriteSheetMediaSource.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) SRG SSR. All rights reserved. + * License information is available from the LICENSE file. + */ +package ch.srgssr.pillarbox.core.business.source + +import androidx.media3.common.MediaItem +import androidx.media3.datasource.TransferListener +import androidx.media3.exoplayer.source.BaseMediaSource +import androidx.media3.exoplayer.source.MediaPeriod +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.SinglePeriodTimeline +import androidx.media3.exoplayer.upstream.Allocator +import ch.srgssr.pillarbox.core.business.integrationlayer.data.SpriteSheet +import kotlin.time.Duration.Companion.milliseconds + +/** + * Sprite sheet media source + * + * @param spriteSheet The [SpriteSheet] to build thumbnails. + */ +class SpriteSheetMediaSource( + private val spriteSheet: SpriteSheet +) : BaseMediaSource() { + + private val mediaItem: MediaItem = MediaItem.Builder() + .setUri(spriteSheet.url) + .setTag(spriteSheet) + .build() + + override fun getMediaItem(): MediaItem { + return mediaItem + } + + override fun canUpdateMediaItem(mediaItem: MediaItem): Boolean { + return this.mediaItem.localConfiguration == mediaItem.localConfiguration + } + + override fun maybeThrowSourceInfoRefreshError() = Unit + + override fun createPeriod(id: MediaSource.MediaPeriodId, allocator: Allocator, startPositionUs: Long): MediaPeriod { + return SpriteSheetMediaPeriod(spriteSheet) + } + + override fun releasePeriod(mediaPeriod: MediaPeriod) { + (mediaPeriod as SpriteSheetMediaPeriod).releasePeriod() + } + + override fun prepareSourceInternal(mediaTransferListener: TransferListener?) { + val duration = spriteSheet.rows * spriteSheet.columns * spriteSheet.interval + val timeline = SinglePeriodTimeline(duration.milliseconds.inWholeMicroseconds, true, false, false, null, getMediaItem()) + refreshSourceInfo(timeline) + } + + override fun releaseSourceInternal() = Unit +}