Skip to content

Commit

Permalink
[media] Refine DecoderBufferAllocator memory budget
Browse files Browse the repository at this point in the history
Now SbMedia budget functions are properly respected when deciding
MediaSource buffer memory limits.

DecoderBuffer::Allocator interface has been refined to remove SbMedia
types, and all Starboard dependency is resolved inside the concrete
DecoderBufferAllocator implementation in //media/starboard.

The logic to determine whether a video is 10 bits has been refined, as
the video mime type is no longer available.

b/322027866
  • Loading branch information
xiaomings committed Jan 10, 2025
1 parent 632168a commit ba5f861
Show file tree
Hide file tree
Showing 10 changed files with 112 additions and 92 deletions.
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

/.github/ # no owners
/cobalt/ # no owners
/media/base/starboard/ # no owners
/media/starboard/ # no owners
/starboard/ # no owners
/components/viz/service/display/starboard/ # no owners
Expand Down
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ files: |
(?x)^(
.github/|
cobalt/|
media/base/starboard/|
media/starboard/|
starboard/|
components/viz/service/display/starboard/|
Expand Down
4 changes: 3 additions & 1 deletion media/base/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,9 @@ source_set("base") {
]
}

if (is_android) {
if (is_cobalt && use_starboard_media) {
sources += [ "starboard/demuxer_memory_limit_starboard.cc" ]
} else if (is_android) {
sources += [ "demuxer_memory_limit_android.cc" ]
} else if (is_castos) {
sources += [ "demuxer_memory_limit_cast.cc" ]
Expand Down
30 changes: 14 additions & 16 deletions media/base/decoder_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,7 @@
#include "media/base/decrypt_config.h"
#include "media/base/media_export.h"
#include "media/base/timestamp_constants.h"

#if BUILDFLAG(USE_STARBOARD_MEDIA)
#include "starboard/media.h"
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
#include "media/base/video_codecs.h"

namespace media {

Expand Down Expand Up @@ -90,12 +87,13 @@ class MEDIA_EXPORT DecoderBuffer
virtual int GetAudioBufferBudget() const = 0;
virtual int GetBufferAlignment() const = 0;
virtual int GetBufferPadding() const = 0;
virtual base::TimeDelta GetBufferGarbageCollectionDurationThreshold() const = 0;
virtual int GetProgressiveBufferBudget(SbMediaVideoCodec codec,
virtual base::TimeDelta GetBufferGarbageCollectionDurationThreshold()
const = 0;
virtual int GetProgressiveBufferBudget(VideoCodec codec,
int resolution_width,
int resolution_height,
int bits_per_pixel) const = 0;
virtual int GetVideoBufferBudget(SbMediaVideoCodec codec,
virtual int GetVideoBufferBudget(VideoCodec codec,
int resolution_width,
int resolution_height,
int bits_per_pixel) const = 0;
Expand All @@ -105,7 +103,7 @@ class MEDIA_EXPORT DecoderBuffer

static void Set(Allocator* allocator);
};
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)

// Allocates buffer with |size| >= 0. |is_key_frame_| will default to false.
explicit DecoderBuffer(size_t size);
Expand Down Expand Up @@ -202,28 +200,28 @@ class MEDIA_EXPORT DecoderBuffer
DCHECK(!end_of_stream());
#if BUILDFLAG(USE_STARBOARD_MEDIA)
return data_;
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
if (read_only_mapping_.IsValid())
return read_only_mapping_.GetMemoryAs<const uint8_t>();
if (writable_mapping_.IsValid())
return writable_mapping_.GetMemoryAs<const uint8_t>();
if (external_memory_)
return external_memory_->span().data();
return data_.get();
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
}

// TODO(sandersd): Remove writable_data(). https://crbug.com/834088
uint8_t* writable_data() const {
#if BUILDFLAG(USE_STARBOARD_MEDIA)
return data_;
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
DCHECK(!end_of_stream());
DCHECK(!read_only_mapping_.IsValid());
DCHECK(!writable_mapping_.IsValid());
DCHECK(!external_memory_);
return data_.get();
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
}

size_t data_size() const {
Expand Down Expand Up @@ -270,12 +268,12 @@ class MEDIA_EXPORT DecoderBuffer
DCHECK_LE(size, size_);
size_ = size;
}
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
bool end_of_stream() const {
return !read_only_mapping_.IsValid() && !writable_mapping_.IsValid() &&
!external_memory_ && !data_;
}
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)

bool is_key_frame() const {
DCHECK(!end_of_stream());
Expand Down Expand Up @@ -329,10 +327,10 @@ class MEDIA_EXPORT DecoderBuffer
// Encoded data, allocated from DecoderBuffer::Allocator.
uint8_t* data_ = nullptr;
size_t allocated_size_ = 0;
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
// Encoded data, if it is stored on the heap.
std::unique_ptr<uint8_t[]> data_;
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)

private:
TimeInfo time_info_;
Expand Down
46 changes: 0 additions & 46 deletions media/base/demuxer_memory_limit_starboard.cc

This file was deleted.

83 changes: 83 additions & 0 deletions media/base/starboard/demuxer_memory_limit_starboard.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright 2022 The Cobalt Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "media/base/demuxer_memory_limit.h"

#include "base/logging.h"
#include "build/build_config.h"
#include "media/base/decoder_buffer.h"
#include "media/base/video_codecs.h"

#if !BUILDFLAG(USE_STARBOARD_MEDIA)
#error "This file only works with Starboard media."
#endif // !BUILDFLAG(USE_STARBOARD_MEDIA)

namespace media {
namespace {

int GetBitsPerPixel(const VideoDecoderConfig& video_config) {
bool is_hdr = false;

if (video_config.codec() == VideoCodec::kVP9) {
if (video_config.profile() == VP9PROFILE_PROFILE2 ||
video_config.profile() == VP9PROFILE_PROFILE3) {
is_hdr = true;
}
} else if (video_config.codec() == VideoCodec::kAV1) {
const VideoColorSpace& color_space = video_config.color_space_info();

if (color_space.primaries >= VideoColorSpace::PrimaryID::BT2020 ||
color_space.transfer >= VideoColorSpace::TransferID::BT2020_10) {
is_hdr = true;
}
}

int bits = is_hdr ? 10 : 8;

LOG(INFO) << "Assume " << bits << " bits for "
<< video_config.AsHumanReadableString();

return bits;
}

} // namespace

size_t GetDemuxerStreamAudioMemoryLimit(
const AudioDecoderConfig* /*audio_config*/) {
return DecoderBuffer::Allocator::GetInstance()->GetAudioBufferBudget();
}

size_t GetDemuxerStreamVideoMemoryLimit(
Demuxer::DemuxerTypes /*demuxer_type*/,
const VideoDecoderConfig* video_config) {
if (!video_config) {
return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget(
VideoCodec::kH264, 1920, 1080, 8);
}

auto codec = video_config->codec();
auto width = video_config->visible_rect().size().width();
auto height = video_config->visible_rect().size().height();
auto bits_per_pixel = GetBitsPerPixel(*video_config);

return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget(
codec, width, height, bits_per_pixel);
}

size_t GetDemuxerMemoryLimit(Demuxer::DemuxerTypes demuxer_type) {
return GetDemuxerStreamAudioMemoryLimit(nullptr) +
GetDemuxerStreamVideoMemoryLimit(demuxer_type, nullptr);
}

} // namespace media
13 changes: 8 additions & 5 deletions media/starboard/decoder_buffer_allocator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <vector>

#include "base/logging.h"
#include "media/base/video_codecs.h"
#include "media/starboard/starboard_utils.h"
#include "starboard/common/allocator.h"
#include "starboard/configuration.h"
Expand Down Expand Up @@ -139,19 +140,21 @@ DecoderBufferAllocator::GetBufferGarbageCollectionDurationThreshold() const {
}

int DecoderBufferAllocator::GetProgressiveBufferBudget(
SbMediaVideoCodec codec,
VideoCodec codec,
int resolution_width,
int resolution_height,
int bits_per_pixel) const {
return SbMediaGetProgressiveBufferBudget(codec, resolution_width,
resolution_height, bits_per_pixel);
return SbMediaGetProgressiveBufferBudget(
MediaVideoCodecToSbMediaVideoCodec(codec), resolution_width,
resolution_height, bits_per_pixel);
}

int DecoderBufferAllocator::GetVideoBufferBudget(SbMediaVideoCodec codec,
int DecoderBufferAllocator::GetVideoBufferBudget(VideoCodec codec,
int resolution_width,
int resolution_height,
int bits_per_pixel) const {
return SbMediaGetVideoBufferBudget(codec, resolution_width, resolution_height,
return SbMediaGetVideoBufferBudget(MediaVideoCodecToSbMediaVideoCodec(codec),
resolution_width, resolution_height,
bits_per_pixel);
}

Expand Down
4 changes: 2 additions & 2 deletions media/starboard/decoder_buffer_allocator.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@ class DecoderBufferAllocator : public DecoderBuffer::Allocator,
int GetBufferAlignment() const override;
int GetBufferPadding() const override;
base::TimeDelta GetBufferGarbageCollectionDurationThreshold() const override;
int GetProgressiveBufferBudget(SbMediaVideoCodec codec,
int GetProgressiveBufferBudget(VideoCodec codec,
int resolution_width,
int resolution_height,
int bits_per_pixel) const override;
int GetVideoBufferBudget(SbMediaVideoCodec codec,
int GetVideoBufferBudget(VideoCodec codec,
int resolution_width,
int resolution_height,
int bits_per_pixel) const override;
Expand Down
19 changes: 0 additions & 19 deletions media/starboard/starboard_utils.cc
Original file line number Diff line number Diff line change
Expand Up @@ -361,25 +361,6 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata(

return sb_media_color_metadata;
}
int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config,
const std::string& mime_type) {
#if BUILDFLAG(USE_STARBOARD_MEDIA)
if (!video_config) {
return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget(
kSbMediaVideoCodecH264, 1920, 1080, 8);
}

auto width = video_config->visible_rect().size().width();
auto height = video_config->visible_rect().size().height();
auto bits_per_pixel = GetBitsPerPixel(mime_type);
auto codec = MediaVideoCodecToSbMediaVideoCodec(video_config->codec());
return DecoderBuffer::Allocator::GetInstance()->GetVideoBufferBudget(
codec, width, height, bits_per_pixel);
#else // BUILDFLAG(USE_STARBOARD_MEDIA)
NOTREACHED();
return 0;
#endif // BUILDFLAG(USE_STARBOARD_MEDIA)
}

std::string ExtractCodecs(const std::string& mime_type) {
static const char kCodecs[] = "codecs=";
Expand Down
3 changes: 0 additions & 3 deletions media/starboard/starboard_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,6 @@ SbMediaColorMetadata MediaToSbMediaColorMetadata(
const absl::optional<gfx::HDRMetadata>& hdr_metadata,
const std::string& mime_type);

int GetSbMediaVideoBufferBudget(const VideoDecoderConfig* video_config,
const std::string& mime_type);

// Extract the value of "codecs" parameter from |mime_type|. It will return
// "avc1.42E01E" for `video/mp4; codecs="avc1.42E01E"`.
// Note that this function assumes that the input is always valid and does
Expand Down

0 comments on commit ba5f861

Please sign in to comment.