From acd1769a87813b3e44aad4012ac80f826ea4bcfa Mon Sep 17 00:00:00 2001 From: Xiaoming Shi Date: Wed, 8 Jan 2025 16:48:47 -0800 Subject: [PATCH] [media] Refine DecoderBufferAllocator memory budget Now SbMedia budget functions are properly respected when deciding MediaSource buffer memory limits. `demuxer_memory_limit_starboard.cc` has been added to //media/base instead of //media/starboard to avoid circular dependency, and to be consistent with the existing Chromium mechanism to branch Demuxer memory limits on different platforms. DecoderBuffer::Allocator interface has been refined to remove SbMedia types, and all Starboard dependency is resolved inside the concrete DecoderBufferAllocator implementation in //media/starboard. The logic to determine whether a video is 10 bits has been refined, as the video mime type is no longer available. b/322027866 --- media/base/BUILD.gn | 4 +- media/base/decoder_buffer.h | 9 +- media/base/demuxer_memory_limit_starboard.cc | 91 ++++++++++++++++++++ media/starboard/decoder_buffer_allocator.cc | 13 +-- media/starboard/decoder_buffer_allocator.h | 4 +- media/starboard/starboard_utils.cc | 19 ---- 6 files changed, 107 insertions(+), 33 deletions(-) create mode 100644 media/base/demuxer_memory_limit_starboard.cc diff --git a/media/base/BUILD.gn b/media/base/BUILD.gn index 62a60695952b..576acf5955d0 100644 --- a/media/base/BUILD.gn +++ b/media/base/BUILD.gn @@ -462,7 +462,9 @@ source_set("base") { ] } - if (is_android) { + if (is_cobalt && use_starboard_media) { + sources += [ "demuxer_memory_limit_starboard.cc" ] + } else if (is_android) { sources += [ "demuxer_memory_limit_android.cc" ] } else if (is_castos) { sources += [ "demuxer_memory_limit_cast.cc" ] diff --git a/media/base/decoder_buffer.h b/media/base/decoder_buffer.h index ec03e6fa1ac2..67d657480e2b 100644 --- a/media/base/decoder_buffer.h +++ b/media/base/decoder_buffer.h @@ -24,10 +24,7 @@ #include "media/base/decrypt_config.h" #include "media/base/media_export.h" #include "media/base/timestamp_constants.h" - -#if BUILDFLAG(USE_STARBOARD_MEDIA) -#include "starboard/media.h" -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) +#include "media/base/video_codecs.h" namespace media { @@ -91,11 +88,11 @@ class MEDIA_EXPORT DecoderBuffer virtual int GetBufferAlignment() const = 0; virtual int GetBufferPadding() const = 0; virtual base::TimeDelta GetBufferGarbageCollectionDurationThreshold() const = 0; - virtual int GetProgressiveBufferBudget(SbMediaVideoCodec codec, + virtual int GetProgressiveBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const = 0; - virtual int GetVideoBufferBudget(SbMediaVideoCodec codec, + virtual int GetVideoBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const = 0; diff --git a/media/base/demuxer_memory_limit_starboard.cc b/media/base/demuxer_memory_limit_starboard.cc new file mode 100644 index 000000000000..7da71bb1127c --- /dev/null +++ b/media/base/demuxer_memory_limit_starboard.cc @@ -0,0 +1,91 @@ +// Copyright 2022 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "media/base/demuxer_memory_limit.h" + +#include "build/build_config.h" +#include "media/base/decoder_buffer.h" +#include "media/base/video_codecs.h" +#include "base/logging.h" + +#if !BUILDFLAG(USE_STARBOARD_MEDIA) +#error "This file only works with Starboard media." +#endif // !BUILDFLAG(USE_STARBOARD_MEDIA) + +namespace media { +namespace { + +int GetBitsPerPixel(const VideoDecoderConfig* video_config) { + DCHECK(video_config); + + if (video_config->codec() == VideoCodec::kH264) { + LOG(INFO) << "H264 encountered, assume 8 bits."; + return 8; + } + + int bits = 8; + + if (video_config->codec() == VideoCodec::kVP9) { + if (video_config->profile() == VP9PROFILE_PROFILE2 || + video_config->profile() == VP9PROFILE_PROFILE3) { + bits = 10; + } + + LOG(INFO) << "VP9 profile " + << (video_config->profile() - VP9PROFILE_PROFILE0) + << " encountered, assume " << bits << " bits."; + return bits; + } + + if (video_config->codec() == VideoCodec::kAV1) { + if (video_config->color_space_info().primaries >= VideoColorSpace::PrimaryID::BT2020 || + video_config->color_space_info().transfer >= VideoColorSpace::TransferID::BT2020_10) { + bits = 10; + } + LOG(INFO) << "AV1 with color space " << video_config->color_space_info().ToString() + << " encountered, assume " << bits << " bits."; + } + + return 8; +} + +} // namespace + +size_t GetDemuxerStreamAudioMemoryLimit( + const AudioDecoderConfig* /*audio_config*/) { + return DecoderBuffer::Allocator::GetInstance()->GetAudioBufferBudget(); +} + +size_t GetDemuxerStreamVideoMemoryLimit( + Demuxer::DemuxerTypes /*demuxer_type*/, + const VideoDecoderConfig* video_config) { + if (!video_config) { + return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( + VideoCodec::kH264, 1920, 1080, 8); + } + + auto codec = video_config->codec(); + auto width = video_config->visible_rect().size().width(); + auto height = video_config->visible_rect().size().height(); + auto bits_per_pixel = GetBitsPerPixel(video_config); + return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( + codec, width, height, bits_per_pixel); +} + +size_t GetDemuxerMemoryLimit(Demuxer::DemuxerTypes demuxer_type) { + return GetDemuxerStreamAudioMemoryLimit(nullptr) + + GetDemuxerStreamVideoMemoryLimit(demuxer_type, nullptr); +} + +} // namespace media diff --git a/media/starboard/decoder_buffer_allocator.cc b/media/starboard/decoder_buffer_allocator.cc index 2dc77690401e..9b6e976b235a 100644 --- a/media/starboard/decoder_buffer_allocator.cc +++ b/media/starboard/decoder_buffer_allocator.cc @@ -18,6 +18,7 @@ #include #include "base/logging.h" +#include "media/base/video_codecs.h" #include "media/starboard/starboard_utils.h" #include "starboard/common/allocator.h" #include "starboard/configuration.h" @@ -139,19 +140,21 @@ DecoderBufferAllocator::GetBufferGarbageCollectionDurationThreshold() const { } int DecoderBufferAllocator::GetProgressiveBufferBudget( - SbMediaVideoCodec codec, + VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const { - return SbMediaGetProgressiveBufferBudget(codec, resolution_width, - resolution_height, bits_per_pixel); + return SbMediaGetProgressiveBufferBudget( + MediaVideoCodecToSbMediaVideoCodec(codec), resolution_width, + resolution_height, bits_per_pixel); } -int DecoderBufferAllocator::GetVideoBufferBudget(SbMediaVideoCodec codec, +int DecoderBufferAllocator::GetVideoBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const { - return SbMediaGetVideoBufferBudget(codec, resolution_width, resolution_height, + return SbMediaGetVideoBufferBudget(MediaVideoCodecToSbMediaVideoCodec(codec), + resolution_width, resolution_height, bits_per_pixel); } diff --git a/media/starboard/decoder_buffer_allocator.h b/media/starboard/decoder_buffer_allocator.h index b3d9ad35677f..dee7e5dc9b99 100644 --- a/media/starboard/decoder_buffer_allocator.h +++ b/media/starboard/decoder_buffer_allocator.h @@ -47,11 +47,11 @@ class DecoderBufferAllocator : public DecoderBuffer::Allocator, int GetBufferAlignment() const override; int GetBufferPadding() const override; base::TimeDelta GetBufferGarbageCollectionDurationThreshold() const override; - int GetProgressiveBufferBudget(SbMediaVideoCodec codec, + int GetProgressiveBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const override; - int GetVideoBufferBudget(SbMediaVideoCodec codec, + int GetVideoBufferBudget(VideoCodec codec, int resolution_width, int resolution_height, int bits_per_pixel) const override; diff --git a/media/starboard/starboard_utils.cc b/media/starboard/starboard_utils.cc index 56495224d212..45f99d68f4cd 100644 --- a/media/starboard/starboard_utils.cc +++ b/media/starboard/starboard_utils.cc @@ -361,25 +361,6 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata( return sb_media_color_metadata; } -int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config, - const std::string& mime_type) { -#if BUILDFLAG(USE_STARBOARD_MEDIA) - if (!video_config) { - return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( - kSbMediaVideoCodecH264, 1920, 1080, 8); - } - - auto width = video_config->visible_rect().size().width(); - auto height = video_config->visible_rect().size().height(); - auto bits_per_pixel = GetBitsPerPixel(mime_type); - auto codec = MediaVideoCodecToSbMediaVideoCodec(video_config->codec()); - return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget( - codec, width, height, bits_per_pixel); -#else // BUILDFLAG(USE_STARBOARD_MEDIA) - NOTREACHED(); - return 0; -#endif // BUILDFLAG(USE_STARBOARD_MEDIA) -} std::string ExtractCodecs(const std::string& mime_type) { static const char kCodecs[] = "codecs=";