From 9ddf1c68c1f330725148f383781bb7372bfae644 Mon Sep 17 00:00:00 2001 From: Xiaoming Shi Date: Wed, 8 Jan 2025 16:48:47 -0800 Subject: [PATCH] [media] Refine DecoderBufferAllocator memory budget Now SbMedia budget functions are properly respected when deciding MediaSource buffer memory limits. DecoderBuffer::Allocator interface has been refined to remove SbMedia types, and all Starboard dependency is resolved inside the concrete DecoderBufferAllocator implementation in //media/starboard. The logic to determine whether a video is 10 bits has been refined, as the video mime type is no longer available. b/322027866 --- media/base/BUILD.gn | 4 +- media/base/decoder_buffer.h | 30 ++++----- media/base/demuxer_memory_limit_starboard.cc | 71 +++++++++++++++++--- media/starboard/decoder_buffer_allocator.cc | 13 ++-- media/starboard/decoder_buffer_allocator.h | 4 +- media/starboard/starboard_utils.cc | 19 ------ media/starboard/starboard_utils.h | 3 - 7 files changed, 87 insertions(+), 57 deletions(-) diff --git a/media/base/BUILD.gn b/media/base/BUILD.gn index 62a60695952b..576acf5955d0 100644 --- a/media/base/BUILD.gn +++ b/media/base/BUILD.gn @@ -462,7 +462,9 @@ source_set("base") { ] } - if (is_android) { + if (is_cobalt && use_starboard_media) { + sources += [ "demuxer_memory_limit_starboard.cc" ] + } else if (is_android) { sources += [ "demuxer_memory_limit_android.cc" ] } else if (is_castos) { sources += [ "demuxer_memory_limit_cast.cc" ] diff --git a/media/base/decoder_buffer.h b/media/base/decoder_buffer.h index ec03e6fa1ac2..e6a6c155247c 100644 --- a/media/base/decoder_buffer.h +++ b/media/base/decoder_buffer.h @@ -24,10 +24,7 @@ #include "media/base/decrypt_config.h" #include "media/base/media_export.h" #include "media/base/timestamp_constants.h" - -#if BUILDFLAG(USE_STARBOARD_MEDIA) -#include "starboard/media.h" -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#include "media/base/video_codecs.h" namespace media { @@ -90,12 +87,13 @@ class MEDIA_EXPORT DecoderBuffer virtual int GetAudioBufferBudget() const = 0; virtual int GetBufferAlignment() const = 0; virtual int GetBufferPadding() const = 0; - virtual base::TimeDelta GetBufferGarbageCollectionDurationThreshold() const = 0; - virtual int GetProgressiveBufferBudget(SbMediaVideoCodec codec, + virtual base::TimeDelta GetBufferGarbageCollectionDurationThreshold() + const = 0; + virtual int GetProgressiveBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const = 0; - virtual int GetVideoBufferBudget(SbMediaVideoCodec codec, + virtual int GetVideoBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const = 0; @@ -105,7 +103,7 @@ class MEDIA_EXPORT DecoderBuffer static void Set(Allocator* allocator); }; -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) // Allocates buffer with |size| >= 0. |is_key_frame_| will default to false. explicit DecoderBuffer(size_t size); @@ -202,7 +200,7 @@ class MEDIA_EXPORT DecoderBuffer DCHECK(!end_of_stream()); #if BUILDFLAG(USE_STARBOARD_MEDIA) return data_; -#else // BUILDFLAG(USE_STARBOARD_MEDIA) +#else // BUILDFLAG(USE_STARBOARD_MEDIA) if (read_only_mapping_.IsValid()) return read_only_mapping_.GetMemoryAs(); if (writable_mapping_.IsValid()) @@ -210,20 +208,20 @@ class MEDIA_EXPORT DecoderBuffer if (external_memory_) return external_memory_->span().data(); return data_.get(); -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) } // TODO(sandersd): Remove writable_data(). https://crbug.com/834088 uint8_t* writable_data() const { #if BUILDFLAG(USE_STARBOARD_MEDIA) return data_; -#else // BUILDFLAG(USE_STARBOARD_MEDIA) +#else // BUILDFLAG(USE_STARBOARD_MEDIA) DCHECK(!end_of_stream()); DCHECK(!read_only_mapping_.IsValid()); DCHECK(!writable_mapping_.IsValid()); DCHECK(!external_memory_); return data_.get(); -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) } size_t data_size() const { @@ -270,12 +268,12 @@ class MEDIA_EXPORT DecoderBuffer DCHECK_LE(size, size_); size_ = size; } -#else // BUILDFLAG(USE_STARBOARD_MEDIA) +#else // BUILDFLAG(USE_STARBOARD_MEDIA) bool end_of_stream() const { return !read_only_mapping_.IsValid() && !writable_mapping_.IsValid() && !external_memory_ && !data_; } -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) bool is_key_frame() const { DCHECK(!end_of_stream()); @@ -329,10 +327,10 @@ class MEDIA_EXPORT DecoderBuffer // Encoded data, allocated from DecoderBuffer::Allocator. uint8_t* data_ = nullptr; size_t allocated_size_ = 0; -#else // BUILDFLAG(USE_STARBOARD_MEDIA) +#else // BUILDFLAG(USE_STARBOARD_MEDIA) // Encoded data, if it is stored on the heap. std::unique_ptr data_; -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) private: TimeInfo time_info_; diff --git a/media/base/demuxer_memory_limit_starboard.cc b/media/base/demuxer_memory_limit_starboard.cc index 188878d8715c..0e10108a2adc 100644 --- a/media/base/demuxer_memory_limit_starboard.cc +++ b/media/base/demuxer_memory_limit_starboard.cc @@ -12,18 +12,58 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !defined(STARBOARD) -#error "This file only works with Cobalt/Starboard." -#endif // !defined(STARBOARD) - #include "media/base/demuxer_memory_limit.h" +#include "base/logging.h" +#include "build/build_config.h" #include "media/base/decoder_buffer.h" -#include "media/base/starboard_utils.h" #include "media/base/video_codecs.h" -#include "base/logging.h" + +#if !BUILDFLAG(USE_STARBOARD_MEDIA) +#error "This file only works with Starboard media." +#endif // !BUILDFLAG(USE_STARBOARD_MEDIA) namespace media { +namespace { + +int GetBitsPerPixel(const VideoDecoderConfig* video_config) { + DCHECK(video_config); + + if (video_config->codec() == VideoCodec::kH264) { + LOG(INFO) << "H264 encountered, assume 8 bits."; + return 8; + } + + int bits = 8; + + if (video_config->codec() == VideoCodec::kVP9) { + if (video_config->profile() == VP9PROFILE_PROFILE2 || + video_config->profile() == VP9PROFILE_PROFILE3) { + bits = 10; + } + + LOG(INFO) << "VP9 profile " + << (video_config->profile() - VP9PROFILE_PROFILE0) + << " encountered, assume " << bits << " bits."; + return bits; + } + + if (video_config->codec() == VideoCodec::kAV1) { + if (video_config->color_space_info().primaries >= + VideoColorSpace::PrimaryID::BT2020 || + video_config->color_space_info().transfer >= + VideoColorSpace::TransferID::BT2020_10) { + bits = 10; + } + LOG(INFO) << "AV1 with color space " + << video_config->color_space_info().ToString() + << " encountered, assume " << bits << " bits."; + } + + return 8; +} + +} // namespace size_t GetDemuxerStreamAudioMemoryLimit( const AudioDecoderConfig* /*audio_config*/) { @@ -32,15 +72,24 @@ size_t GetDemuxerStreamAudioMemoryLimit( size_t GetDemuxerStreamVideoMemoryLimit( Demuxer::DemuxerTypes /*demuxer_type*/, - const VideoDecoderConfig* video_config, - const std::string& mime_type) { - return static_cast( - GetSbMediaVideoBufferBudget(video_config, mime_type)); + const VideoDecoderConfig* video_config) { + if (!video_config) { + return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( + VideoCodec::kH264, 1920, 1080, 8); + } + + auto codec = video_config->codec(); + auto width = video_config->visible_rect().size().width(); + auto height = video_config->visible_rect().size().height(); + auto bits_per_pixel = GetBitsPerPixel(video_config); + + return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( + codec, width, height, bits_per_pixel); } size_t GetDemuxerMemoryLimit(Demuxer::DemuxerTypes demuxer_type) { return GetDemuxerStreamAudioMemoryLimit(nullptr) + - GetDemuxerStreamVideoMemoryLimit(demuxer_type, nullptr, ""); + GetDemuxerStreamVideoMemoryLimit(demuxer_type, nullptr); } } // namespace media diff --git a/media/starboard/decoder_buffer_allocator.cc b/media/starboard/decoder_buffer_allocator.cc index 2dc77690401e..9b6e976b235a 100644 --- a/media/starboard/decoder_buffer_allocator.cc +++ b/media/starboard/decoder_buffer_allocator.cc @@ -18,6 +18,7 @@ #include #include "base/logging.h" +#include "media/base/video_codecs.h" #include "media/starboard/starboard_utils.h" #include "starboard/common/allocator.h" #include "starboard/configuration.h" @@ -139,19 +140,21 @@ DecoderBufferAllocator::GetBufferGarbageCollectionDurationThreshold() const { } int DecoderBufferAllocator::GetProgressiveBufferBudget( - SbMediaVideoCodec codec, + VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const { - return SbMediaGetProgressiveBufferBudget(codec, resolution_width, - resolution_height, bits_per_pixel); + return SbMediaGetProgressiveBufferBudget( + MediaVideoCodecToSbMediaVideoCodec(codec), resolution_width, + resolution_height, bits_per_pixel); } -int DecoderBufferAllocator::GetVideoBufferBudget(SbMediaVideoCodec codec, +int DecoderBufferAllocator::GetVideoBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const { - return SbMediaGetVideoBufferBudget(codec, resolution_width, resolution_height, + return SbMediaGetVideoBufferBudget(MediaVideoCodecToSbMediaVideoCodec(codec), + resolution_width, resolution_height, bits_per_pixel); } diff --git a/media/starboard/decoder_buffer_allocator.h b/media/starboard/decoder_buffer_allocator.h index b3d9ad35677f..dee7e5dc9b99 100644 --- a/media/starboard/decoder_buffer_allocator.h +++ b/media/starboard/decoder_buffer_allocator.h @@ -47,11 +47,11 @@ class DecoderBufferAllocator : public DecoderBuffer::Allocator, int GetBufferAlignment() const override; int GetBufferPadding() const override; base::TimeDelta GetBufferGarbageCollectionDurationThreshold() const override; - int GetProgressiveBufferBudget(SbMediaVideoCodec codec, + int GetProgressiveBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const override; - int GetVideoBufferBudget(SbMediaVideoCodec codec, + int GetVideoBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const override; diff --git a/media/starboard/starboard_utils.cc b/media/starboard/starboard_utils.cc index 56495224d212..45f99d68f4cd 100644 --- a/media/starboard/starboard_utils.cc +++ b/media/starboard/starboard_utils.cc @@ -361,25 +361,6 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata( return sb_media_color_metadata; } -int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, - const std::string& mime_type) { -#if BUILDFLAG(USE_STARBOARD_MEDIA) - if (!video_config) { - return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( - kSbMediaVideoCodecH264, 1920, 1080, 8); - } - - auto width = video_config->visible_rect().size().width(); - auto height = video_config->visible_rect().size().height(); - auto bits_per_pixel = GetBitsPerPixel(mime_type); - auto codec = MediaVideoCodecToSbMediaVideoCodec(video_config->codec()); - return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( - codec, width, height, bits_per_pixel); -#else // BUILDFLAG(USE_STARBOARD_MEDIA) - NOTREACHED(); - return 0; -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) -} std::string ExtractCodecs(const std::string& mime_type) { static const char kCodecs[] = "codecs="; diff --git a/media/starboard/starboard_utils.h b/media/starboard/starboard_utils.h index 8f2a174efc03..497721c50119 100644 --- a/media/starboard/starboard_utils.h +++ b/media/starboard/starboard_utils.h @@ -47,9 +47,6 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata( const absl::optional& hdr_metadata, const std::string& mime_type); -int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, - const std::string& mime_type); - // Extract the value of "codecs" parameter from |mime_type|. It will return // "avc1.42E01E" for `video/mp4; codecs="avc1.42E01E"`. // Note that this function assumes that the input is always valid and does