[android] Handle mediacodec callback with a handler thread

starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java

@@ -31,6 +31,8 @@
 import android.media.MediaFormat;
 import android.os.Build;
 import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import dev.cobalt.util.Log;
@@ -93,6 +95,9 @@ class MediaCodecBridge {
   // which would cause GC cycles long enough to impact playback.
   private final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
 
+  private Handler mHandler = null;
+  private HandlerThread mCallbackThread = null;
+
   // Type of bitrate adjustment for video encoder.
   public enum BitrateAdjustmentTypes {
     // No adjustment - video encoder has no known bitrate problem.
@@ -451,7 +456,8 @@ private MediaCodecBridge(
       MediaCodec mediaCodec,
       String mime,
       BitrateAdjustmentTypes bitrateAdjustmentType,
-      int tunnelModeAudioSessionId) {
+      int tunnelModeAudioSessionId,
+      boolean useCallbackThread) {
     if (mediaCodec == null) {
       throw new IllegalArgumentException();
     }
@@ -461,6 +467,11 @@ private MediaCodecBridge(
     mLastPresentationTimeUs = 0;
     mFlushed = true;
     mBitrateAdjustmentType = bitrateAdjustmentType;
+    if (useCallbackThread) {
+      mCallbackThread = new HandlerThread("MediaCodec:Callback:Handler");
+      mCallbackThread.start();
+      mHandler = new Handler(mCallbackThread.getLooper());
+    }
     mCallback =
         new MediaCodec.Callback() {
           @Override
@@ -525,7 +536,7 @@ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
             }
           }
         };
-    mMediaCodec.setCallback(mCallback);
+    mMediaCodec.setCallback(mCallback, mHandler);
 
     // TODO: support OnFrameRenderedListener for non tunnel mode
     if (tunnelModeAudioSessionId != -1) {
@@ -555,6 +566,7 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
       int sampleRate,
       int channelCount,
       MediaCrypto crypto,
+      boolean useCallbackThread,
       @Nullable byte[] configurationData) {
     if (decoderName.equals("")) {
       Log.e(TAG, "Invalid decoder name.");
@@ -573,7 +585,12 @@ public static MediaCodecBridge createAudioMediaCodecBridge(
     }
     MediaCodecBridge bridge =
         new MediaCodecBridge(
-            nativeMediaCodecBridge, mediaCodec, mime, BitrateAdjustmentTypes.NO_ADJUSTMENT, -1);
+            nativeMediaCodecBridge,
+            mediaCodec,
+            mime,
+            BitrateAdjustmentTypes.NO_ADJUSTMENT,
+            -1,
+            useCallbackThread);
 
     MediaFormat mediaFormat = createAudioFormat(mime, sampleRate, channelCount);
 
@@ -618,6 +635,7 @@ public static void createVideoMediaCodecBridge(
       MediaCrypto crypto,
       ColorInfo colorInfo,
       int tunnelModeAudioSessionId,
+      boolean useCallbackThread,
       CreateMediaCodecBridgeResult outCreateMediaCodecBridgeResult) {
     MediaCodec mediaCodec = null;
     outCreateMediaCodecBridgeResult.mMediaCodecBridge = null;
@@ -671,7 +689,8 @@ public static void createVideoMediaCodecBridge(
             mediaCodec,
             mime,
             BitrateAdjustmentTypes.NO_ADJUSTMENT,
-            tunnelModeAudioSessionId);
+            tunnelModeAudioSessionId,
+            useCallbackThread);
     MediaFormat mediaFormat =
         createVideoDecoderFormat(mime, widthHint, heightHint, videoCapabilities);
 
@@ -806,6 +825,11 @@ public void release() {
       Log.e(TAG, "Cannot release media codec", e);
     }
     mMediaCodec = null;
+    if (mCallbackThread != null) {
+      mCallbackThread.quitSafely();
+      mCallbackThread = null;
+      mHandler = null;
+    }
   }
 
   @SuppressWarnings("unused")

starboard/android/shared/audio_decoder.cc

@@ -70,12 +70,14 @@ void* IncrementPointerByBytes(void* pointer, int offset) {
 }  // namespace
 
 AudioDecoder::AudioDecoder(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread)
     : audio_stream_info_(audio_stream_info),
       sample_type_(GetSupportedSampleType()),
       output_sample_rate_(audio_stream_info.samples_per_second),
       output_channel_count_(audio_stream_info.number_of_channels),
-      drm_system_(static_cast<DrmSystem*>(drm_system)) {
+      drm_system_(static_cast<DrmSystem*>(drm_system)),
+      use_mediacodec_callback_thread_(use_mediacodec_callback_thread) {
   if (!InitializeCodec()) {
     SB_LOG(ERROR) << "Failed to initialize audio decoder.";
   }
@@ -186,7 +188,8 @@ void AudioDecoder::Reset() {
 
 bool AudioDecoder::InitializeCodec() {
   SB_DCHECK(!media_decoder_);
-  media_decoder_.reset(new MediaDecoder(this, audio_stream_info_, drm_system_));
+  media_decoder_.reset(new MediaDecoder(this, audio_stream_info_, drm_system_,
+                                        use_mediacodec_callback_thread_));
   if (media_decoder_->is_valid()) {
     if (error_cb_) {
       media_decoder_->Initialize(

starboard/android/shared/audio_decoder.h

@@ -45,7 +45,8 @@ class AudioDecoder
       AudioStreamInfo;
 
   AudioDecoder(const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool use_mediacodec_callback_thread);
   ~AudioDecoder() override;
 
   void Initialize(const OutputCB& output_cb, const ErrorCB& error_cb) override;
@@ -83,6 +84,10 @@ class AudioDecoder
 
   DrmSystem* drm_system_;
 
+  // Set mediacodec callback with a handler on another thread to avoid running
+  // callbacks on the main thread and being blocked by other main thread tasks.
+  const bool use_mediacodec_callback_thread_;
+
   OutputCB output_cb_;
   ErrorCB error_cb_;
   ConsumedCB consumed_cb_;

starboard/android/shared/audio_renderer_passthrough.cc

@@ -75,14 +75,15 @@ int ParseAc3SyncframeAudioSampleCount(const uint8_t* buffer, int size) {
 
 AudioRendererPassthrough::AudioRendererPassthrough(
     const AudioStreamInfo& audio_stream_info,
-    SbDrmSystem drm_system)
+    SbDrmSystem drm_system,
+    bool use_mediacodec_callback_thread)
     : audio_stream_info_(audio_stream_info) {
   SB_DCHECK(audio_stream_info_.codec == kSbMediaAudioCodecAc3 ||
             audio_stream_info_.codec == kSbMediaAudioCodecEac3);
   if (SbDrmSystemIsValid(drm_system)) {
     SB_LOG(INFO) << "Creating AudioDecoder as decryptor.";
-    scoped_ptr<AudioDecoder> audio_decoder(
-        new AudioDecoder(audio_stream_info, drm_system));
+    scoped_ptr<AudioDecoder> audio_decoder(new AudioDecoder(
+        audio_stream_info, drm_system, use_mediacodec_callback_thread));
     if (audio_decoder->is_valid()) {
       decoder_.reset(audio_decoder.release());
     }

starboard/android/shared/audio_renderer_passthrough.h

@@ -53,7 +53,8 @@ class AudioRendererPassthrough
       AudioStreamInfo;
 
   AudioRendererPassthrough(const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system);
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread);
   ~AudioRendererPassthrough() override;
 
   bool is_valid() const { return decoder_ != nullptr; }

starboard/android/shared/media_codec_bridge.cc

@@ -158,7 +158,8 @@ Java_dev_cobalt_media_MediaCodecBridge_nativeOnMediaCodecOutputFormatChanged(
 scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
     const AudioStreamInfo& audio_stream_info,
     Handler* handler,
-    jobject j_media_crypto) {
+    jobject j_media_crypto,
+    bool use_callback_thread) {
   bool is_passthrough = false;
   const char* mime =
       SupportedAudioCodecToMimeType(audio_stream_info.codec, &is_passthrough);
@@ -193,11 +194,11 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateAudioMediaCodecBridge(
       new MediaCodecBridge(handler));
   jobject j_media_codec_bridge = env->CallStaticObjectMethodOrAbort(
       "dev/cobalt/media/MediaCodecBridge", "createAudioMediaCodecBridge",
-      "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;"
+      "(JLjava/lang/String;Ljava/lang/String;IILandroid/media/MediaCrypto;Z"
       "[B)Ldev/cobalt/media/MediaCodecBridge;",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),
       j_decoder_name.Get(), audio_stream_info.samples_per_second,
-      audio_stream_info.number_of_channels, j_media_crypto,
+      audio_stream_info.number_of_channels, j_media_crypto, use_callback_thread,
       configuration_data.Get());
 
   if (!j_media_codec_bridge) {
@@ -227,6 +228,7 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateVideoMediaCodecBridge(
     bool require_software_codec,
     int tunnel_mode_audio_session_id,
     bool force_big_endian_hdr_metadata,
+    bool use_callback_thread,
     std::string* error_message) {
   SB_DCHECK(error_message);
   SB_DCHECK(max_width.has_engaged() == max_height.has_engaged());
@@ -316,14 +318,14 @@ scoped_ptr<MediaCodecBridge> MediaCodecBridge::CreateVideoMediaCodecBridge(
       "(JLjava/lang/String;Ljava/lang/String;IIIIILandroid/view/Surface;"
       "Landroid/media/MediaCrypto;"
       "Ldev/cobalt/media/MediaCodecBridge$ColorInfo;"
-      "I"
+      "IZ"
       "Ldev/cobalt/media/MediaCodecBridge$CreateMediaCodecBridgeResult;)"
       "V",
       reinterpret_cast<jlong>(native_media_codec_bridge.get()), j_mime.Get(),
       j_decoder_name.Get(), width_hint, height_hint, fps,
       max_width.value_or(-1), max_height.value_or(-1), j_surface,
       j_media_crypto, j_color_info.Get(), tunnel_mode_audio_session_id,
-      j_create_media_codec_bridge_result.Get());
+      use_callback_thread, j_create_media_codec_bridge_result.Get());
 
   jobject j_media_codec_bridge = env->CallObjectMethodOrAbort(
       j_create_media_codec_bridge_result.Get(), "mediaCodecBridge",

starboard/android/shared/media_codec_bridge.h

@@ -147,7 +147,8 @@ class MediaCodecBridge {
   static scoped_ptr<MediaCodecBridge> CreateAudioMediaCodecBridge(
       const AudioStreamInfo& audio_stream_info,
       Handler* handler,
-      jobject j_media_crypto);
+      jobject j_media_crypto,
+      bool use_callback_thread);
 
   // `max_width` and `max_height` can be set to positive values to specify the
   // maximum resolutions the video can be adapted to.
@@ -173,6 +174,7 @@ class MediaCodecBridge {
       bool require_software_codec,
       int tunnel_mode_audio_session_id,
       bool force_big_endian_hdr_metadata,
+      bool use_callback_thread,
       std::string* error_message);
 
   ~MediaCodecBridge();

starboard/android/shared/media_decoder.cc

@@ -76,7 +76,8 @@ const char* GetDecoderName(SbMediaType media_type) {
 
 MediaDecoder::MediaDecoder(Host* host,
                            const AudioStreamInfo& audio_stream_info,
-                           SbDrmSystem drm_system)
+                           SbDrmSystem drm_system,
+                           bool use_mediacodec_callback_thread)
     : media_type_(kSbMediaTypeAudio),
       host_(host),
       drm_system_(static_cast<DrmSystem*>(drm_system)),
@@ -87,7 +88,7 @@ MediaDecoder::MediaDecoder(Host* host,
   jobject j_media_crypto = drm_system_ ? drm_system_->GetMediaCrypto() : NULL;
   SB_DCHECK(!drm_system_ || j_media_crypto);
   media_codec_bridge_ = MediaCodecBridge::CreateAudioMediaCodecBridge(
-      audio_stream_info, this, j_media_crypto);
+      audio_stream_info, this, j_media_crypto, use_mediacodec_callback_thread);
   if (!media_codec_bridge_) {
     SB_LOG(ERROR) << "Failed to create audio media codec bridge.";
     return;
@@ -118,6 +119,7 @@ MediaDecoder::MediaDecoder(Host* host,
                            const FrameRenderedCB& frame_rendered_cb,
                            int tunnel_mode_audio_session_id,
                            bool force_big_endian_hdr_metadata,
+                           bool use_mediacodec_callback_thread,
                            std::string* error_message)
     : media_type_(kSbMediaTypeVideo),
       host_(host),
@@ -135,7 +137,8 @@ MediaDecoder::MediaDecoder(Host* host,
       video_codec, width_hint, height_hint, fps, max_width, max_height, this,
       j_output_surface, j_media_crypto, color_metadata, require_secured_decoder,
       require_software_codec, tunnel_mode_audio_session_id,
-      force_big_endian_hdr_metadata, error_message);
+      force_big_endian_hdr_metadata, use_mediacodec_callback_thread,
+      error_message);
   if (!media_codec_bridge_) {
     SB_LOG(ERROR) << "Failed to create video media codec bridge with error: "
                   << *error_message;

starboard/android/shared/media_decoder.h

@@ -79,7 +79,8 @@ class MediaDecoder
 
   MediaDecoder(Host* host,
                const AudioStreamInfo& audio_stream_info,
-               SbDrmSystem drm_system);
+               SbDrmSystem drm_system,
+               bool use_mediacodec_callback_thread);
   MediaDecoder(Host* host,
                SbMediaVideoCodec video_codec,
                // `width_hint` and `height_hint` are used to create the Android
@@ -97,6 +98,7 @@ class MediaDecoder
                const FrameRenderedCB& frame_rendered_cb,
                int tunnel_mode_audio_session_id,
                bool force_big_endian_hdr_metadata,
+               bool use_mediacodec_callback_thread,
                std::string* error_message);
   ~MediaDecoder();
 

starboard/android/shared/media_is_video_supported.cc

@@ -92,6 +92,10 @@ bool SbMediaIsVideoSupported(SbMediaVideoCodec video_codec,
                                      false)) {
       MaxMediaCodecOutputBuffersLookupTable::GetInstance()->SetEnabled(false);
     }
+
+    if (!mime_type->ValidateBoolParameter("mediacodeccallbackthread")) {
+      return false;
+    }
   }
 
   if (must_support_tunnel_mode && decode_to_texture_required) {