Bug 1912543 - Fix requestVideoFrameCallback with suspended/invisible …

…videos. r=media-playback-reviewers,padenot

When a tab is backgrounded, or on some platforms, when the window is
fully covered, full decoding of a video for playback is disabled. The
video timeline still advances, but there are no valid frames for
presentation. Similarly, if an HTMLVideoElement is in the DOM tree but
marked as invisible (e.g. 'display: none;' is set), we also cease full
decoding of a video. This is the VideoDecodeMode::Suspend mode.

Chrome and Safari both continue to honour requestVideoFrameCallback when
the video element is invisible in a foreground tab. Conversely, when we
are in a backgrounded tab, Chrome suspends rVFC callbacks, while Safari
continues.

Given that we suspend requestAnimationFrame callbacks similar to Chrome
for backgrounded tabs, this patch matches our behaviour with Chrome.

The standard does not discuss the implications of visibility and
background/foreground on rVFC. We have filed an issue requesting for
clarification, and that can tracked at:

WICG/video-rvfc#92

Differential Revision: https://phabricator.services.mozilla.com/D220274

dom/html/HTMLMediaElement.cpp

@@ -7533,8 +7533,9 @@ void HTMLMediaElement::GetEMEInfo(dom::EMEDebugInfo& aInfo) {
 
 void HTMLMediaElement::NotifyDecoderActivityChanges() const {
   if (mDecoder) {
-    mDecoder->NotifyOwnerActivityChanged(IsActuallyInvisible(),
-                                         IsInComposedDoc());
+    mDecoder->NotifyOwnerActivityChanged(
+        IsActuallyInvisible(), IsInComposedDoc(),
+        OwnerDoc()->IsInBackgroundWindow(), HasPendingCallbacks());
   }
 }
 

dom/html/HTMLMediaElement.h

@@ -1414,6 +1414,10 @@ class HTMLMediaElement : public nsGenericHTMLElement,
   // changes its status of being used in the Picture-in-Picture mode.
   void UpdateMediaControlAfterPictureInPictureModeChanged();
 
+  // Return true if the element has pending callbacks that should prevent the
+  // suspension of video playback.
+  virtual bool HasPendingCallbacks() const { return false; }
+
   // The current decoder. Load() has been called on this decoder.
   // At most one of mDecoder and mSrcStream can be non-null.
   RefPtr<MediaDecoder> mDecoder;

dom/html/HTMLVideoElement.cpp

@@ -749,6 +749,13 @@ void HTMLVideoElement::TakeVideoFrameRequestCallbacks(
     return;
   }
 
+  // If we have got a dummy frame, then we must have suspended decoding and have
+  // no actual frame to present. This should only happen if we raced on
+  // requesting a callback, and the media state machine advancing.
+  if (NS_WARN_IF(frameSize.IsEmpty())) {
+    return;
+  }
+
   // If we have already displayed the expected frame, we need to make the
   // display time match the presentation time to indicate it is already
   // complete.
@@ -775,12 +782,29 @@ void HTMLVideoElement::TakeVideoFrameRequestCallbacks(
 
   mLastPresentedFrameID = frameID;
   mVideoFrameRequestManager.Take(aCallbacks);
+
+  NS_DispatchToMainThread(NewRunnableMethod(
+      "HTMLVideoElement::FinishedVideoFrameRequestCallbacks", this,
+      &HTMLVideoElement::FinishedVideoFrameRequestCallbacks));
+}
+
+void HTMLVideoElement::FinishedVideoFrameRequestCallbacks() {
+  // After we have executed the rVFC and rAF callbacks, we need to check whether
+  // or not we have scheduled more. If we did not, then we need to notify the
+  // decoder, because it may be the only thing keeping the decoder fully active.
+  if (!HasPendingCallbacks()) {
+    NotifyDecoderActivityChanges();
+  }
 }
 
 uint32_t HTMLVideoElement::RequestVideoFrameCallback(
     VideoFrameRequestCallback& aCallback, ErrorResult& aRv) {
+  bool hasPending = HasPendingCallbacks();
   uint32_t handle = 0;
   aRv = mVideoFrameRequestManager.Schedule(aCallback, &handle);
+  if (!hasPending && HasPendingCallbacks()) {
+    NotifyDecoderActivityChanges();
+  }
   return handle;
 }
 
@@ -789,7 +813,9 @@ bool HTMLVideoElement::IsVideoFrameCallbackCancelled(uint32_t aHandle) {
 }
 
 void HTMLVideoElement::CancelVideoFrameCallback(uint32_t aHandle) {
-  mVideoFrameRequestManager.Cancel(aHandle);
+  if (mVideoFrameRequestManager.Cancel(aHandle) && !HasPendingCallbacks()) {
+    NotifyDecoderActivityChanges();
+  }
 }
 
 }  // namespace mozilla::dom

dom/html/HTMLVideoElement.h

@@ -193,6 +193,10 @@ class HTMLVideoElement final : public HTMLMediaElement {
  private:
   void ResetState() override;
 
+  bool HasPendingCallbacks() const final {
+    return !mVideoFrameRequestManager.IsEmpty();
+  }
+
   VideoFrameRequestManager mVideoFrameRequestManager;
   layers::ContainerFrameID mLastPresentedFrameID =
       layers::kContainerFrameID_Invalid;
@@ -206,6 +210,7 @@ class HTMLVideoElement final : public HTMLMediaElement {
                                       VideoFrameCallbackMetadata& aMd,
                                       nsTArray<VideoFrameRequest>& aCallbacks);
   bool IsVideoFrameCallbackCancelled(uint32_t aHandle);
+  void FinishedVideoFrameRequestCallbacks();
 
  private:
   static void MapAttributesIntoRule(MappedDeclarationsBuilder&);

dom/media/MediaDecoder.cpp

@@ -140,10 +140,13 @@ void MediaDecoder::InitStatics() {
 NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter)
 
 void MediaDecoder::NotifyOwnerActivityChanged(bool aIsOwnerInvisible,
-                                              bool aIsOwnerConnected) {
+                                              bool aIsOwnerConnected,
+                                              bool aIsOwnerInBackground,
+                                              bool aHasOwnerPendingCallbacks) {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
-  SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected);
+  SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected,
+                       aIsOwnerInBackground, aHasOwnerPendingCallbacks);
 
   NotifyCompositor();
 }
@@ -242,6 +245,8 @@ MediaDecoder::MediaDecoder(MediaDecoderInit& aInit)
       mFiredMetadataLoaded(false),
       mIsOwnerInvisible(false),
       mIsOwnerConnected(false),
+      mIsOwnerInBackground(false),
+      mHasOwnerPendingCallbacks(false),
       mForcedHidden(false),
       mHasSuspendTaint(aInit.mHasSuspendTaint),
       mShouldResistFingerprinting(
@@ -1171,10 +1176,14 @@ void MediaDecoder::NotifyCompositor() {
 }
 
 void MediaDecoder::SetElementVisibility(bool aIsOwnerInvisible,
-                                        bool aIsOwnerConnected) {
+                                        bool aIsOwnerConnected,
+                                        bool aIsOwnerInBackground,
+                                        bool aHasOwnerPendingCallbacks) {
   MOZ_ASSERT(NS_IsMainThread());
   mIsOwnerInvisible = aIsOwnerInvisible;
   mIsOwnerConnected = aIsOwnerConnected;
+  mIsOwnerInBackground = aIsOwnerInBackground;
+  mHasOwnerPendingCallbacks = aHasOwnerPendingCallbacks;
   mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility());
   UpdateVideoDecodeMode();
 }
@@ -1233,6 +1242,14 @@ void MediaDecoder::UpdateVideoDecodeMode() {
     return;
   }
 
+  // Don't suspend elements that have pending rVFC callbacks.
+  if (mHasOwnerPendingCallbacks && !mIsOwnerInBackground) {
+    LOG("UpdateVideoDecodeMode(), set Normal because the element has pending "
+        "callbacks while in foreground.");
+    mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal);
+    return;
+  }
+
   // If mForcedHidden is set, suspend the video decoder anyway.
   if (mForcedHidden) {
     LOG("UpdateVideoDecodeMode(), set Suspend because the element is forced to "

dom/media/MediaDecoder.h

@@ -170,8 +170,10 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> {
   virtual void Play();
 
   // Notify activity of the decoder owner is changed.
-  virtual void NotifyOwnerActivityChanged(bool aIsOwnerInvisible,
-                                          bool aIsOwnerConnected);
+  void NotifyOwnerActivityChanged(bool aIsOwnerInvisible,
+                                  bool aIsOwnerConnected,
+                                  bool aIsOwnerInBackground,
+                                  bool aHasOwnerPendingCallbacks);
 
   // Pause video playback.
   virtual void Pause();
@@ -336,8 +338,9 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> {
   bool CanPlayThrough();
 
   // Called from HTMLMediaElement when owner document activity changes
-  virtual void SetElementVisibility(bool aIsOwnerInvisible,
-                                    bool aIsOwnerConnected);
+  void SetElementVisibility(bool aIsOwnerInvisible, bool aIsOwnerConnected,
+                            bool aIsOwnerInBackground,
+                            bool aHasOwnerPendingCallbacks);
 
   // Force override the visible state to hidden.
   // Called from HTMLMediaElement when testing of video decode suspend from
@@ -633,6 +636,12 @@ class MediaDecoder : public DecoderDoctorLifeLogger<MediaDecoder> {
   // https://dom.spec.whatwg.org/#connected
   bool mIsOwnerConnected;
 
+  // True if the owner element is in a backgrounded tab/window.
+  bool mIsOwnerInBackground;
+
+  // True if the owner element has pending rVFC callbacks.
+  bool mHasOwnerPendingCallbacks;
+
   // If true, forces the decoder to be considered hidden.
   bool mForcedHidden;