[android] Improve video/audio seeking via flush MediaCodec

1. Currently, Cobalt resets MediaCodec during seek, which is very slow on some atv devices.
2. This improvement cuts down the InBufferSeek duration by approximately 40%.
3. Adjusting the process to flush MediaCodec rather than dismantling and re-establishing it during seek operations prevents the platform from mistakenly concluding that playback has ended.

b/320568573

starboard/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java

@@ -833,6 +833,8 @@ public void release() {
   @UsedByNative
   private boolean start() {
     try {
+      String codecName = mMediaCodec.getName();
+      Log.w(TAG, "calling MediaCodec.start() on " + codecName);
       mMediaCodec.start();
     } catch (IllegalStateException | IllegalArgumentException e) {
       Log.e(TAG, "Cannot start the media codec", e);
@@ -883,6 +885,8 @@ private void updateOperatingRate() {
   @UsedByNative
   private int flush() {
     try {
+      String codecName = mMediaCodec.getName();
+      Log.w(TAG, "calling MediaCodec.flush() on " + codecName);
       mFlushed = true;
       mMediaCodec.flush();
     } catch (Exception e) {

starboard/android/shared/audio_decoder.cc

@@ -136,6 +136,8 @@ void AudioDecoder::WriteEndOfStream() {
 
   if (media_decoder_) {
     media_decoder_->WriteEndOfStream();
+
+    stream_ended_.store(true);
   }
 }
 
@@ -167,15 +169,21 @@ void AudioDecoder::Reset() {
   SB_DCHECK(BelongsToCurrentThread());
   SB_DCHECK(output_cb_);
 
-  media_decoder_.reset();
-  audio_frame_discarder_.Reset();
+  // If it is not end of the stream, then try to flush if we can,
+  // otherwise completely recreate and reconfigure the codec.
+  if (stream_ended_.load() || !media_decoder_->TryFlush()) {
+    SB_LOG(ERROR) << "Failed to flush/start codec, reset codec";
+    media_decoder_.reset();
 
-  if (!InitializeCodec()) {
-    // TODO: Communicate this failure to our clients somehow.
-    SB_LOG(ERROR) << "Failed to initialize codec after reset.";
+    if (!InitializeCodec()) {
+      // TODO: Communicate this failure to our clients somehow.
+      SB_LOG(ERROR) << "Failed to initialize codec after reset.";
+    }
   }
+  audio_frame_discarder_.Reset();
 
   consumed_cb_ = nullptr;
+  stream_ended_.store(false);
 
   while (!decoded_audios_.empty()) {
     decoded_audios_.pop();

starboard/android/shared/audio_decoder.h

@@ -92,6 +92,8 @@ class AudioDecoder
 
   AudioFrameDiscarder audio_frame_discarder_;
   scoped_ptr<MediaDecoder> media_decoder_;
+
+  atomic_bool stream_ended_;
 };
 
 }  // namespace shared

starboard/android/shared/media_codec_bridge.cc

@@ -446,6 +446,11 @@ void MediaCodecBridge::SetPlaybackRate(double playback_rate) {
       j_media_codec_bridge_, "setPlaybackRate", "(D)V", playback_rate);
 }
 
+bool MediaCodecBridge::Start() {
+  return JniEnvExt::Get()->CallBooleanMethodOrAbort(j_media_codec_bridge_,
+                                                    "start", "()Z") == JNI_TRUE;
+}
+
 jint MediaCodecBridge::Flush() {
   return JniEnvExt::Get()->CallIntMethodOrAbort(j_media_codec_bridge_, "flush",
                                                 "()I");

starboard/android/shared/media_codec_bridge.h

@@ -198,6 +198,7 @@ class MediaCodecBridge {
   void ReleaseOutputBufferAtTimestamp(jint index, jlong render_timestamp_ns);
 
   void SetPlaybackRate(double playback_rate);
+  bool Start();
   jint Flush();
   FrameSize GetOutputSize();
   AudioOutputFormatResult GetAudioOutputFormat();

starboard/android/shared/media_decoder.cc

@@ -177,6 +177,7 @@ void MediaDecoder::Initialize(const ErrorCB& error_cb) {
   SB_DCHECK(!error_cb_);
 
   error_cb_ = error_cb;
+  find_key_frame_after_flushing_.store(false);
 
   if (error_occurred_) {
     Schedule(std::bind(error_cb_, error_, error_message_));
@@ -208,6 +209,17 @@ void MediaDecoder::WriteInputBuffers(const InputBuffers& input_buffers) {
   ScopedLock scoped_lock(mutex_);
   bool need_signal = pending_tasks_.empty();
   for (const auto& input_buffer : input_buffers) {
+    // It is important that the input data after start()
+    // or flush() starts at a suitable stream boundary:
+    // the first frame must be a key frame.
+    if (find_key_frame_after_flushing_.load() &&
+        media_type_ == kSbMediaTypeVideo) {
+      if (!input_buffer->video_sample_info().is_key_frame) {
+        continue;
+      } else {
+        find_key_frame_after_flushing_.store(false);
+      }
+    }
     pending_tasks_.push_back(Event(input_buffer));
     number_of_pending_tasks_.increment();
   }
@@ -279,6 +291,10 @@ void MediaDecoder::DecoderThreadFunc() {
       }
 
       for (;;) {
+        while (number_of_pending_tasks_.load() < pending_tasks.size()) {
+          // flush pending tasks added before TryFlush()
+          pending_tasks.pop_front();
+        }
         bool can_process_input =
             pending_queue_input_buffer_task_ ||
             (!pending_tasks.empty() && !input_buffer_indices.empty());
@@ -299,6 +315,10 @@ void MediaDecoder::DecoderThreadFunc() {
     std::vector<DequeueOutputResult> dequeue_output_results;
 
     while (!destroying_.load()) {
+      while (number_of_pending_tasks_.load() < pending_tasks.size()) {
+        // flush pending tasks added before TryFlush()
+        pending_tasks.pop_front();
+      }
       bool has_input =
           pending_queue_input_buffer_task_ ||
           (!pending_tasks.empty() && !input_buffer_indices.empty());
@@ -594,6 +614,7 @@ void MediaDecoder::OnMediaCodecError(bool is_recoverable,
 }
 
 void MediaDecoder::OnMediaCodecInputBufferAvailable(int buffer_index) {
+  // TODO(borongchen): check input_buffer_indices_
   if (media_type_ == kSbMediaTypeVideo && first_call_on_handler_thread_) {
     // Set the thread priority of the Handler thread to dispatch the async
     // decoder callbacks to high.
@@ -644,6 +665,36 @@ void MediaDecoder::OnMediaCodecFrameRendered(int64_t frame_timestamp) {
   frame_rendered_cb_(frame_timestamp);
 }
 
+bool MediaDecoder::TryFlush() {
+  if (is_valid()) {
+    host_->OnFlushing();
+    jint status = media_codec_bridge_->Flush();
+    if (status != MEDIA_CODEC_OK) {
+      SB_LOG(ERROR) << "Failed to flush media codec.";
+      return false;
+    }
+    // As the codec is configured in asynchronous mode, call start()
+    // after flush has returned to resume codec operations.
+    if (!media_codec_bridge_->Start()) {
+      SB_LOG(ERROR) << "Failed to start media codec.";
+      return false;
+    }
+  }
+
+  // Find the key frame after flush, as the first frame must be a key frame.
+  if (media_type_ == kSbMediaTypeVideo) {
+    find_key_frame_after_flushing_.store(true);
+  }
+
+  // Clear up all pending tasks.
+  ScopedLock scoped_lock(mutex_);
+  number_of_pending_tasks_.store(0);
+  pending_queue_input_buffer_task_ = nullopt_t();
+  stream_ended_.store(false);
+
+  return true;
+}
+
 }  // namespace shared
 }  // namespace android
 }  // namespace starboard

starboard/android/shared/media_decoder.h

@@ -113,6 +113,11 @@ class MediaDecoder
 
   bool is_valid() const { return media_codec_bridge_ != NULL; }
 
+  // Try to flush this media codec. Returns true on success,
+  // false on failure. If this returns false, let
+  // AudioDecoder/VideoDecoder reset MediaDecoder.
+  bool TryFlush();
+
  private:
   struct Event {
     enum Type {
@@ -189,6 +194,8 @@ class MediaDecoder
 
   atomic_bool destroying_;
 
+  atomic_bool find_key_frame_after_flushing_;
+
   optional<QueueInputBufferTask> pending_queue_input_buffer_task_;
 
   atomic_int32_t number_of_pending_tasks_;

starboard/android/shared/video_decoder.cc

@@ -599,16 +599,26 @@ void VideoDecoder::WriteEndOfStream() {
 void VideoDecoder::Reset() {
   SB_DCHECK(BelongsToCurrentThread());
 
-  TeardownCodec();
+  // If it is not end of the stream, then try to flush if we can,
+  // otherwise completely recreate and reconfigure the codec.
+  if (end_of_stream_written_ || !media_decoder_->TryFlush()) {
+    SB_LOG(ERROR) << "Failed to flush/start codec, reset codec";
+    TeardownCodec();
+
+    output_format_ = starboard::nullopt;
+
+    // If the codec is kSbMediaVideoCodecAv1,
+    // set video_fps_ to 0 will call InitializeCodec(),
+    // which we do not need if flush the codec.
+    video_fps_ = 0;
+  }
   CancelPendingJobs();
 
   tunnel_mode_prerolling_.store(true);
   tunnel_mode_frame_rendered_.store(false);
   input_buffer_written_ = 0;
   decoded_output_frames_ = 0;
-  output_format_ = starboard::nullopt;
   end_of_stream_written_ = false;
-  video_fps_ = 0;
   pending_input_buffers_.clear();
 
   // TODO: We rely on VideoRenderAlgorithmTunneled::Seek() to be called inside

starboard/shared/starboard/player/filter/adaptive_audio_decoder_internal.cc

@@ -45,6 +45,7 @@ AdaptiveAudioDecoder::AdaptiveAudioDecoder(
 
 AdaptiveAudioDecoder::~AdaptiveAudioDecoder() {
   Reset();
+  TeardownAudioDecoder();
 }
 
 void AdaptiveAudioDecoder::Initialize(const OutputCB& output_cb,
@@ -146,7 +147,13 @@ void AdaptiveAudioDecoder::Reset() {
   SB_DCHECK(BelongsToCurrentThread());
 
   if (audio_decoder_) {
-    TeardownAudioDecoder();
+    // If it is not end of the stream, then try to flush if we can,
+    // otherwise completely recreate and reconfigure the codec.
+    if (!stream_ended_) {
+      audio_decoder_->Reset();
+    } else {
+      TeardownAudioDecoder();
+    }
   }
   CancelPendingJobs();
   while (!decoded_audios_.empty()) {