From 24de2e3980753e82fb068cd365e33641c43396d7 Mon Sep 17 00:00:00 2001 From: Andrew Osmond Date: Wed, 28 Aug 2024 17:10:35 +0000 Subject: [PATCH] Bug 1912543 - Fix requestVideoFrameCallback with suspended/invisible videos. r=media-playback-reviewers,padenot When a tab is backgrounded, or on some platforms, when the window is fully covered, full decoding of a video for playback is disabled. The video timeline still advances, but there are no valid frames for presentation. Similarly, if an HTMLVideoElement is in the DOM tree but marked as invisible (e.g. 'display: none;' is set), we also cease full decoding of a video. This is the VideoDecodeMode::Suspend mode. Chrome and Safari both continue to honour requestVideoFrameCallback when the video element is invisible in a foreground tab. Conversely, when we are in a backgrounded tab, Chrome suspends rVFC callbacks, while Safari continues. Given that we suspend requestAnimationFrame callbacks similar to Chrome for backgrounded tabs, this patch matches our behaviour with Chrome. The standard does not discuss the implications of visibility and background/foreground on rVFC. We have filed an issue requesting for clarification, and that can tracked at: https://github.com/WICG/video-rvfc/issues/92 Differential Revision: https://phabricator.services.mozilla.com/D220274 --- dom/html/HTMLMediaElement.cpp | 5 +++-- dom/html/HTMLMediaElement.h | 4 ++++ dom/html/HTMLVideoElement.cpp | 28 +++++++++++++++++++++++++++- dom/html/HTMLVideoElement.h | 5 +++++ dom/media/MediaDecoder.cpp | 23 ++++++++++++++++++++--- dom/media/MediaDecoder.h | 17 +++++++++++++---- 6 files changed, 72 insertions(+), 10 deletions(-) diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 11b23f73194c0..b7bf758f43b8c 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -7533,8 +7533,9 @@ void HTMLMediaElement::GetEMEInfo(dom::EMEDebugInfo& aInfo) { void HTMLMediaElement::NotifyDecoderActivityChanges() const { if (mDecoder) { - mDecoder->NotifyOwnerActivityChanged(IsActuallyInvisible(), - IsInComposedDoc()); + mDecoder->NotifyOwnerActivityChanged( + IsActuallyInvisible(), IsInComposedDoc(), + OwnerDoc()->IsInBackgroundWindow(), HasPendingCallbacks()); } } diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index 119cd61278613..402946724073f 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -1414,6 +1414,10 @@ class HTMLMediaElement : public nsGenericHTMLElement, // changes its status of being used in the Picture-in-Picture mode. void UpdateMediaControlAfterPictureInPictureModeChanged(); + // Return true if the element has pending callbacks that should prevent the + // suspension of video playback. + virtual bool HasPendingCallbacks() const { return false; } + // The current decoder. Load() has been called on this decoder. // At most one of mDecoder and mSrcStream can be non-null. RefPtr mDecoder; diff --git a/dom/html/HTMLVideoElement.cpp b/dom/html/HTMLVideoElement.cpp index 1022fefe98f8e..99d1942047416 100644 --- a/dom/html/HTMLVideoElement.cpp +++ b/dom/html/HTMLVideoElement.cpp @@ -749,6 +749,13 @@ void HTMLVideoElement::TakeVideoFrameRequestCallbacks( return; } + // If we have got a dummy frame, then we must have suspended decoding and have + // no actual frame to present. This should only happen if we raced on + // requesting a callback, and the media state machine advancing. + if (NS_WARN_IF(frameSize.IsEmpty())) { + return; + } + // If we have already displayed the expected frame, we need to make the // display time match the presentation time to indicate it is already // complete. @@ -775,12 +782,29 @@ void HTMLVideoElement::TakeVideoFrameRequestCallbacks( mLastPresentedFrameID = frameID; mVideoFrameRequestManager.Take(aCallbacks); + + NS_DispatchToMainThread(NewRunnableMethod( + "HTMLVideoElement::FinishedVideoFrameRequestCallbacks", this, + &HTMLVideoElement::FinishedVideoFrameRequestCallbacks)); +} + +void HTMLVideoElement::FinishedVideoFrameRequestCallbacks() { + // After we have executed the rVFC and rAF callbacks, we need to check whether + // or not we have scheduled more. If we did not, then we need to notify the + // decoder, because it may be the only thing keeping the decoder fully active. + if (!HasPendingCallbacks()) { + NotifyDecoderActivityChanges(); + } } uint32_t HTMLVideoElement::RequestVideoFrameCallback( VideoFrameRequestCallback& aCallback, ErrorResult& aRv) { + bool hasPending = HasPendingCallbacks(); uint32_t handle = 0; aRv = mVideoFrameRequestManager.Schedule(aCallback, &handle); + if (!hasPending && HasPendingCallbacks()) { + NotifyDecoderActivityChanges(); + } return handle; } @@ -789,7 +813,9 @@ bool HTMLVideoElement::IsVideoFrameCallbackCancelled(uint32_t aHandle) { } void HTMLVideoElement::CancelVideoFrameCallback(uint32_t aHandle) { - mVideoFrameRequestManager.Cancel(aHandle); + if (mVideoFrameRequestManager.Cancel(aHandle) && !HasPendingCallbacks()) { + NotifyDecoderActivityChanges(); + } } } // namespace mozilla::dom diff --git a/dom/html/HTMLVideoElement.h b/dom/html/HTMLVideoElement.h index ad8086a328695..bd3f99d8e9f8b 100644 --- a/dom/html/HTMLVideoElement.h +++ b/dom/html/HTMLVideoElement.h @@ -193,6 +193,10 @@ class HTMLVideoElement final : public HTMLMediaElement { private: void ResetState() override; + bool HasPendingCallbacks() const final { + return !mVideoFrameRequestManager.IsEmpty(); + } + VideoFrameRequestManager mVideoFrameRequestManager; layers::ContainerFrameID mLastPresentedFrameID = layers::kContainerFrameID_Invalid; @@ -206,6 +210,7 @@ class HTMLVideoElement final : public HTMLMediaElement { VideoFrameCallbackMetadata& aMd, nsTArray& aCallbacks); bool IsVideoFrameCallbackCancelled(uint32_t aHandle); + void FinishedVideoFrameRequestCallbacks(); private: static void MapAttributesIntoRule(MappedDeclarationsBuilder&); diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index c1f635b42933d..6f3e43d3af21e 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -140,10 +140,13 @@ void MediaDecoder::InitStatics() { NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter) void MediaDecoder::NotifyOwnerActivityChanged(bool aIsOwnerInvisible, - bool aIsOwnerConnected) { + bool aIsOwnerConnected, + bool aIsOwnerInBackground, + bool aHasOwnerPendingCallbacks) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); - SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected); + SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected, + aIsOwnerInBackground, aHasOwnerPendingCallbacks); NotifyCompositor(); } @@ -242,6 +245,8 @@ MediaDecoder::MediaDecoder(MediaDecoderInit& aInit) mFiredMetadataLoaded(false), mIsOwnerInvisible(false), mIsOwnerConnected(false), + mIsOwnerInBackground(false), + mHasOwnerPendingCallbacks(false), mForcedHidden(false), mHasSuspendTaint(aInit.mHasSuspendTaint), mShouldResistFingerprinting( @@ -1171,10 +1176,14 @@ void MediaDecoder::NotifyCompositor() { } void MediaDecoder::SetElementVisibility(bool aIsOwnerInvisible, - bool aIsOwnerConnected) { + bool aIsOwnerConnected, + bool aIsOwnerInBackground, + bool aHasOwnerPendingCallbacks) { MOZ_ASSERT(NS_IsMainThread()); mIsOwnerInvisible = aIsOwnerInvisible; mIsOwnerConnected = aIsOwnerConnected; + mIsOwnerInBackground = aIsOwnerInBackground; + mHasOwnerPendingCallbacks = aHasOwnerPendingCallbacks; mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility()); UpdateVideoDecodeMode(); } @@ -1233,6 +1242,14 @@ void MediaDecoder::UpdateVideoDecodeMode() { return; } + // Don't suspend elements that have pending rVFC callbacks. + if (mHasOwnerPendingCallbacks && !mIsOwnerInBackground) { + LOG("UpdateVideoDecodeMode(), set Normal because the element has pending " + "callbacks while in foreground."); + mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); + return; + } + // If mForcedHidden is set, suspend the video decoder anyway. if (mForcedHidden) { LOG("UpdateVideoDecodeMode(), set Suspend because the element is forced to " diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index 4d31f221b7f9c..dd0056060a2e1 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -170,8 +170,10 @@ class MediaDecoder : public DecoderDoctorLifeLogger { virtual void Play(); // Notify activity of the decoder owner is changed. - virtual void NotifyOwnerActivityChanged(bool aIsOwnerInvisible, - bool aIsOwnerConnected); + void NotifyOwnerActivityChanged(bool aIsOwnerInvisible, + bool aIsOwnerConnected, + bool aIsOwnerInBackground, + bool aHasOwnerPendingCallbacks); // Pause video playback. virtual void Pause(); @@ -336,8 +338,9 @@ class MediaDecoder : public DecoderDoctorLifeLogger { bool CanPlayThrough(); // Called from HTMLMediaElement when owner document activity changes - virtual void SetElementVisibility(bool aIsOwnerInvisible, - bool aIsOwnerConnected); + void SetElementVisibility(bool aIsOwnerInvisible, bool aIsOwnerConnected, + bool aIsOwnerInBackground, + bool aHasOwnerPendingCallbacks); // Force override the visible state to hidden. // Called from HTMLMediaElement when testing of video decode suspend from @@ -633,6 +636,12 @@ class MediaDecoder : public DecoderDoctorLifeLogger { // https://dom.spec.whatwg.org/#connected bool mIsOwnerConnected; + // True if the owner element is in a backgrounded tab/window. + bool mIsOwnerInBackground; + + // True if the owner element has pending rVFC callbacks. + bool mHasOwnerPendingCallbacks; + // If true, forces the decoder to be considered hidden. bool mForcedHidden;