Cherry pick PR #2044: [android] Handle mediacodec callback with a han…

…dler thread (#2277) Refer to the original PR: #2044 Now the MediaCodec callbacks and the activity lifecycle callbacks are called on the main thread. When pausing YouTube, the MediaCodec callbacks may be blocked until the lifecycle callbacks are done. This may cause frame drops on a resource-limited device when the MediaCodec callback to notify a newly decoded frame is blocked and we run out of decoded frames. Running the MediaCodec callbacks on a handler thread could avoid this problem. Change-Id: I5ffdb1f5a582c3d01964b3f98c99d7aed211674a b/316008643 Co-authored-by: Stone <[email protected]>
youtube · Jan 26, 2024 · 9ce6b86 · 9ce6b86
1 parent 4a2d4b3
commit 9ce6b86
Show file tree

Hide file tree

Showing 13 changed files with 120 additions and 41 deletions.
diff --git a/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java b/starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java
@@ -31,6 +31,8 @@
 import android.media.MediaFormat;
 import android.os.Build;
 import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import dev.cobalt.util.Log;
@@ -93,6 +95,9 @@ class MediaCodecBridge {
   // which would cause GC cycles long enough to impact playback.
   private final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
 
+  private Handler mHandler = null;
+  private HandlerThread mCallbackThread = null;
+
   // Type of bitrate adjustment for video encoder.
   public enum BitrateAdjustmentTypes {
     // No adjustment - video encoder has no known bitrate problem.
@@ -451,7 +456,8 @@ private MediaCodecBridge(
       MediaCodec mediaCodec,
       String mime,
       BitrateAdjustmentTypes bitrateAdjustmentType,
-      int tunnelModeAudioSessionId) {
+      int tunnelModeAudioSessionId,
+      boolean useCallbackThread) {
     if (mediaCodec == null) {
       throw new IllegalArgumentException();
     }
@@ -461,6 +467,11 @@ private MediaCodecBridge(
     mLastPresentationTimeUs = 0;
     mFlushed = true;
     mBitrateAdjustmentType = bitrateAdjustmentType;
+    if (useCallbackThread) {
+      mCallbackThread = new HandlerThread("MediaCodec:Callback:Handler");
+      mCallbackThread.start();
+      mHandler = new Handler(mCallbackThread.getLooper());
+    }
     mCallback =
         new MediaCodec.Callback() {
           @Override
@@ -525,7 +536,7 @@ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
             }
           }
         };
-    mMediaCodec.setCallback(mCallback);
+    mMediaCodec.setCallback(mCallback, mHandler);
 
     // TODO: support OnFrameRenderedListener for non tunnel mode
     if (tunnelModeAudioSessionId != -1) {
@@ -555,6 +566,7 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
       int sampleRate,
       int channelCount,
       MediaCrypto crypto,
+      boolean useCallbackThread,
       @Nullable byte[] configurationData) {
     if (decoderName.equals("")) {
       Log.e(TAG, "Invalid decoder name.");
@@ -573,7 +585,12 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
     }
     MediaCodecBridge bridge =
         new MediaCodecBridge(
-            nativeMediaCodecBridge, mediaCodec, mime, BitrateAdjustmentTypes.NO_ADJUSTMENT, -1);
+            nativeMediaCodecBridge,
+            mediaCodec,
+            mime,
+            BitrateAdjustmentTypes.NO_ADJUSTMENT,
+            -1,
+            useCallbackThread);
 
     MediaFormat mediaFormat = createAudioFormat(mime, sampleRate, channelCount);
 
@@ -618,6 +635,7 @@ public static void createVideoMediaCodecBridge(
       MediaCrypto crypto,
       ColorInfo colorInfo,
       int tunnelModeAudioSessionId,
+      boolean useCallbackThread,
       CreateMediaCodecBridgeResult outCreateMediaCodecBridgeResult) {
     MediaCodec mediaCodec = null;
     outCreateMediaCodecBridgeResult.mMediaCodecBridge = null;
@@ -671,7 +689,8 @@ public static void createVideoMediaCodecBridge(
             mediaCodec,
             mime,
             BitrateAdjustmentTypes.NO_ADJUSTMENT,
-            tunnelModeAudioSessionId);
+            tunnelModeAudioSessionId,
+            useCallbackThread);
     MediaFormat mediaFormat =
         createVideoDecoderFormat(mime, widthHint, heightHint, videoCapabilities);
 
@@ -806,6 +825,11 @@ public void release() {
       Log.e(TAG, "Cannot release media codec", e);
     }
     mMediaCodec = null;
+    if (mCallbackThread != null) {
+      mCallbackThread.quitSafely();
+      mCallbackThread = null;
+      mHandler = null;
+    }
   }
 
   @SuppressWarnings("unused")

diff --git a/starboard/android/shared/audio_decoder.cc b/starboard/android/shared/audio_decoder.cc
@@ -70,12 +70,14 @@ void* IncrementPointerByBytes(void* pointer, int offset) {
 }  // namespace
 
 AudioDecoder::AudioDecoder(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread)
     : audio_stream_info_(audio_stream_info),
       sample_type_(GetSupportedSampleType()),
       output_sample_rate_(audio_stream_info.samples_per_second),
       output_channel_count_(audio_stream_info.number_of_channels),
-      drm_system_(static_cast<DrmSystem*>(drm_system)) {
+      drm_system_(static_cast<DrmSystem*>(drm_system)),
+      use_mediacodec_callback_thread_(use_mediacodec_callback_thread) {
   if (!InitializeCodec()) {
     SB_LOG(ERROR) << "Failed to initialize audio decoder.";
   }
@@ -186,7 +188,8 @@ void AudioDecoder::Reset() {
 
 bool AudioDecoder::InitializeCodec() {
   SB_DCHECK(!media_decoder_);
-  media_decoder_.reset(new MediaDecoder(this, audio_stream_info_, drm_system_));
+  media_decoder_.reset(new MediaDecoder(this, audio_stream_info_, drm_system_,
+                                        use_mediacodec_callback_thread_));
   if (media_decoder_->is_valid()) {
     if (error_cb_) {
       media_decoder_->Initialize(

diff --git a/starboard/android/shared/audio_decoder.h b/starboard/android/shared/audio_decoder.h
@@ -45,7 +45,8 @@ class AudioDecoder
       AudioStreamInfo;
 
   AudioDecoder(const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool use_mediacodec_callback_thread);
   ~AudioDecoder() override;
 
   void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override;
@@ -83,6 +84,10 @@ class AudioDecoder
 
   DrmSystem* drm_system_;
 
+  // Set mediacodec callback with a handler on another thread to avoid running
+  // callbacks on the main thread and being blocked by other main thread tasks.
+  const bool use_mediacodec_callback_thread_;
+
   OutputCB output_cb_;
   ErrorCB error_cb_;
   ConsumedCB consumed_cb_;

diff --git a/starboard/android/shared/audio_renderer_passthrough.cc b/starboard/android/shared/audio_renderer_passthrough.cc
@@ -75,15 +75,16 @@ int ParseAc3SyncframeAudioSampleCount(const uint8_t* buffer, int size) {
 AudioRendererPassthrough::AudioRendererPassthrough(
     const AudioStreamInfo& audio_stream_info,
     SbDrmSystem drm_system,
-    bool enable_audio_device_callback)
+    bool enable_audio_device_callback,
+    bool use_mediacodec_callback_thread)
     : audio_stream_info_(audio_stream_info),
       enable_audio_device_callback_(enable_audio_device_callback) {
   SB_DCHECK(audio_stream_info_.codec == kSbMediaAudioCodecAc3 ||
             audio_stream_info_.codec == kSbMediaAudioCodecEac3);
   if (SbDrmSystemIsValid(drm_system)) {
     SB_LOG(INFO) << "Creating AudioDecoder as decryptor.";
-    scoped_ptr<AudioDecoder> audio_decoder(
-        new AudioDecoder(audio_stream_info, drm_system));
+    scoped_ptr<AudioDecoder> audio_decoder(new AudioDecoder(
+        audio_stream_info, drm_system, use_mediacodec_callback_thread));
     if (audio_decoder->is_valid()) {
       decoder_.reset(audio_decoder.release());
     }

diff --git a/starboard/android/shared/audio_renderer_passthrough.h b/starboard/android/shared/audio_renderer_passthrough.h
@@ -55,7 +55,8 @@ class AudioRendererPassthrough
 
   AudioRendererPassthrough(const AudioStreamInfo& audio_stream_info,
                            SbDrmSystem drm_system,
-                           bool enable_audio_device_callback);
+                           bool enable_audio_device_callback,
+                           bool use_mediacodec_callback_thread);
   ~AudioRendererPassthrough() override;
 
   bool is_valid() const { return decoder_ != nullptr; }

diff --git a/starboard/android/shared/media_codec_bridge.cc b/starboard/android/shared/media_codec_bridge.cc
@@ -158,7 +158,8 @@ Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecOutputFormatChanged(
 scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
     const AudioStreamInfo& audio_stream_info,
     Handler* handler,
-    jobject j_media_crypto) {
+    jobject j_media_crypto,
+    bool use_callback_thread) {
   bool is_passthrough = false;
   const char* mime =
       SupportedAudioCodecToMimeType(audio_stream_info.codec, &is_passthrough);
@@ -193,11 +194,11 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
       new MediaCodecBridge(handler));
   jobject j_media_codec_bridge = env->CallStaticObjectMethodOrAbort(
       "dev/cobalt/media/MediaCodecBridge", "createAudioMediaCodecBridge",
-      "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;"
+      "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;Z"
       "[B)Ldev/cobalt/media/MediaCodecBridge;",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),
       j_decoder_name.Get(), audio_stream_info.samples_per_second,
-      audio_stream_info.number_of_channels, j_media_crypto,
+      audio_stream_info.number_of_channels, j_media_crypto, use_callback_thread,
       configuration_data.Get());
 
   if (!j_media_codec_bridge) {
@@ -228,6 +229,7 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateVideoMediaCodecBridge(
     int tunnel_mode_audio_session_id,
     bool force_big_endian_hdr_metadata,
     bool force_improved_support_check,
+    bool use_callback_thread,
     std::string* error_message) {
   SB_DCHECK(error_message);
   SB_DCHECK(max_width.has_engaged() == max_height.has_engaged());
@@ -317,14 +319,14 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateVideoMediaCodecBridge(
       "(JLjava/lang/String;Ljava/lang/String;IIIIILandroid/view/Surface;"
       "Landroid/media/MediaCrypto;"
       "Ldev/cobalt/media/MediaCodecBridge$ColorInfo;"
-      "I"
+      "IZ"
       "Ldev/cobalt/media/MediaCodecBridge$CreateMediaCodecBridgeResult;)"
       "V",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),
       j_decoder_name.Get(), width_hint, height_hint, fps,
       max_width.value_or(-1), max_height.value_or(-1), j_surface,
       j_media_crypto, j_color_info.Get(), tunnel_mode_audio_session_id,
-      j_create_media_codec_bridge_result.Get());
+      use_callback_thread, j_create_media_codec_bridge_result.Get());
 
   jobject j_media_codec_bridge = env->CallObjectMethodOrAbort(
       j_create_media_codec_bridge_result.Get(), "mediaCodecBridge",

diff --git a/starboard/android/shared/media_codec_bridge.h b/starboard/android/shared/media_codec_bridge.h
@@ -147,7 +147,8 @@ class MediaCodecBridge {
   static scoped_ptr<MediaCodecBridge> CreateAudioMediaCodecBridge(
       const AudioStreamInfo& audio_stream_info,
       Handler* handler,
-      jobject j_media_crypto);
+      jobject j_media_crypto,
+      bool use_callback_thread);
 
   // `max_width` and `max_height` can be set to positive values to specify the
   // maximum resolutions the video can be adapted to.
@@ -174,6 +175,7 @@ class MediaCodecBridge {
       int tunnel_mode_audio_session_id,
       bool force_big_endian_hdr_metadata,
       bool force_improved_support_check,
+      bool use_callback_thread,
       std::string* error_message);
 
   ~MediaCodecBridge();

diff --git a/starboard/android/shared/media_decoder.cc b/starboard/android/shared/media_decoder.cc
@@ -76,7 +76,8 @@ const char* GetDecoderName(SbMediaType media_type) {
 
 MediaDecoder::MediaDecoder(Host* host,
                            const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread)
     : media_type_(kSbMediaTypeAudio),
       host_(host),
       drm_system_(static_cast<DrmSystem*>(drm_system)),
@@ -87,7 +88,7 @@ MediaDecoder::MediaDecoder(Host* host,
   jobject j_media_crypto = drm_system_ ? drm_system_->GetMediaCrypto() : NULL;
   SB_DCHECK(!drm_system_ || j_media_crypto);
   media_codec_bridge_ = MediaCodecBridge::CreateAudioMediaCodecBridge(
-      audio_stream_info, this, j_media_crypto);
+      audio_stream_info, this, j_media_crypto, use_mediacodec_callback_thread);
   if (!media_codec_bridge_) {
     SB_LOG(ERROR) << "Failed to create audio media codec bridge.";
     return;
@@ -119,6 +120,7 @@ MediaDecoder::MediaDecoder(Host* host,
                            int tunnel_mode_audio_session_id,
                            bool force_big_endian_hdr_metadata,
                            bool force_improved_support_check,
+                           bool use_mediacodec_callback_thread,
                            std::string* error_message)
     : media_type_(kSbMediaTypeVideo),
       host_(host),
@@ -137,7 +139,7 @@ MediaDecoder::MediaDecoder(Host* host,
       j_output_surface, j_media_crypto, color_metadata, require_secured_decoder,
       require_software_codec, tunnel_mode_audio_session_id,
       force_big_endian_hdr_metadata, force_improved_support_check,
-      error_message);
+      use_mediacodec_callback_thread, error_message);
   if (!media_codec_bridge_) {
     SB_LOG(ERROR) << "Failed to create video media codec bridge with error: "
                   << *error_message;

diff --git a/starboard/android/shared/media_decoder.h b/starboard/android/shared/media_decoder.h
@@ -79,7 +79,8 @@ class MediaDecoder
 
   MediaDecoder(Host* host,
                const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool use_mediacodec_callback_thread);
   MediaDecoder(Host* host,
                SbMediaVideoCodec video_codec,
                // `width_hint` and `height_hint` are used to create the Android
@@ -98,6 +99,7 @@ class MediaDecoder
                int tunnel_mode_audio_session_id,
                bool force_big_endian_hdr_metadata,
                bool force_improved_support_check,
+               bool use_mediacodec_callback_thread,
                std::string* error_message);
   ~MediaDecoder();
 

diff --git a/starboard/android/shared/media_is_video_supported.cc b/starboard/android/shared/media_is_video_supported.cc
@@ -105,6 +105,10 @@ bool SbMediaIsVideoSupported(SbMediaVideoCodec video_codec,
                                      false)) {
       MaxMediaCodecOutputBuffersLookupTable::GetInstance()->SetEnabled(false);
     }
+
+    if (!mime_type->ValidateBoolParameter("mediacodeccallbackthread")) {
+      return false;
+    }
   }
 
   if (must_support_tunnel_mode && decode_to_texture_required) {