Implement RTCRtpEncodingParameters.scaleResolutionDownTo.

Spec: https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-scaleresolutiondownto Wires up scaleResolutionDownTo[1] to JS behind RuntimeEnabled flag "RTCRtpScaleResolutionDownTo". This is implemented in third_party/webrtc where it is called `requested_resolution`. WPTs are added to test basic functionality, including getting the resolution we expect, changing it on the fly, it being orientation agnostic and throwing on invalid parameters. The tests use small resolutions like 120x60 for fast ramp up even on slow bots (sending HD tends to trigger initial frame dropping and slow BW ramp up). The following test coverage is NOT included yet, but will be added in follow up CL(s): - Simulcast tests: to be written in a separate CL for reviewability. - scaleTo maintaining aspect ratio: blocked on a WebRTC fix. [1] w3c/webrtc-extensions#221 Bug: chromium:363544347 Change-Id: If930ffd686d073d2eb239763e9ea9c1390fbcef1 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5828607 Reviewed-by: Dominik Röttsches <[email protected]> Commit-Queue: Henrik Boström <[email protected]> Reviewed-by: Harald Alvestrand <[email protected]> Cr-Commit-Position: refs/heads/main@{#1355122}
web-platform-tests · Sep 13, 2024 · 85dc128 · 85dc128
1 parent e6f0175
commit 85dc128
Showing 1 changed file with 240 additions and 0 deletions.
diff --git a/webrtc-extensions/RTCRtpEncodingParameters-scaleResolutionDownTo.https.html b/webrtc-extensions/RTCRtpEncodingParameters-scaleResolutionDownTo.https.html
@@ -0,0 +1,240 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>RTCRtpEncodingParameters scaleResolutionDownTo attribute</title>
+<meta name="timeout" content="long">
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>
+'use strict';
+
+// Creates a track that can be resized with `track.resize(w,h)`.
+function createResizableTrack(width, height) {
+  // Draw to a canvas with a 30 fps interval.
+  const canvas = Object.assign(
+      document.createElement('canvas'), {width, height});
+  const ctx = canvas.getContext('2d');
+  ctx.fillStyle = 'rgb(255,0,0)';
+  const interval = setInterval(() => {
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
+  }, 1000 / 30);
+  // Capture the canvas and add/modify reize() and stop() methods.
+  const stream = canvas.captureStream();
+  const [track] = stream.getTracks();
+  track.resize = (w, h) => {
+    canvas.width = w;
+    canvas.height = h;
+  };
+  const nativeStop = track.stop;
+  track.stop = () => {
+    clearInterval(interval);
+    nativeStop.apply(track);
+  };
+  return track;
+}
+
+async function getLastEncodedResolution(pc) {
+  const report = await pc.getStats();
+  for (const stats of report.values()) {
+    if (stats.type != 'outbound-rtp') {
+      continue;
+    }
+    if (!stats.frameWidth || !stats.frameWidth) {
+      // The resolution is missing until the first frame has been encoded.
+      break;
+    }
+    return { width: stats.frameWidth, height: stats.frameHeight };
+  }
+  return { width: null, height: null };
+}
+
+async function waitForFrameWithResolution(t, pc, width, height) {
+  let resolution;
+  do {
+    resolution = await getLastEncodedResolution(pc);
+    await new Promise(r => t.step_timeout(r, 50));
+  } while (resolution.width != width || resolution.height != height);
+}
+
+promise_test(async t => {
+  const pc = new RTCPeerConnection();
+  t.add_cleanup(() => pc.close());
+
+  const track = createResizableTrack(120, 60);
+  t.add_cleanup(() => track.stop());
+  assert_throws_dom('OperationError', () => {
+    pc.addTransceiver(track, {
+          sendEncodings:[{
+              scaleResolutionDownTo: undefined,
+          }, {
+              scaleResolutionDownTo: { maxWidth: 120, maxHeight: 60 }
+          }]
+        });
+  });
+}, `addTransceiver: Specifying scaling on some but not all encodings throws`);
+
+promise_test(async t => {
+  const pc = new RTCPeerConnection();
+  t.add_cleanup(() => pc.close());
+
+  const track = createResizableTrack(120, 60);
+  t.add_cleanup(() => track.stop());
+  const {sender} = pc.addTransceiver(track, {sendEncodings:[{},{}]});
+
+  const params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownTo = undefined;
+  params.encodings[1].scaleResolutionDownTo = { maxWidth: 120, maxHeight: 60 };
+  const p = sender.setParameters(params);
+
+  promise_rejects_dom(t, 'InvalidModificationError', p);
+}, `setParameters: Specifying scaling on some but not all encodings throws`);
+
+promise_test(async t => {
+  const pc1 = new RTCPeerConnection();
+  t.add_cleanup(() => pc1.close());
+  const pc2 = new RTCPeerConnection();
+  t.add_cleanup(() => pc2.close());
+  pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
+  pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
+
+  const track = createResizableTrack(120, 60);
+  t.add_cleanup(() => track.stop());
+
+  pc1.addTransceiver(track, {
+        sendEncodings:[{
+            scaleResolutionDownBy: 2.0,
+            scaleResolutionDownTo: { maxWidth: 120, maxHeight: 60 }
+        }]
+      });
+
+  await pc1.setLocalDescription();
+  await pc2.setRemoteDescription(pc1.localDescription);
+  await pc2.setLocalDescription();
+  await pc1.setRemoteDescription(pc2.localDescription);
+
+  await waitForFrameWithResolution(t, pc1, 120, 60);
+}, `addTransceiver: scaleResolutionDownBy is ignored when ` +
+   `scaleResolutionDownTo is specified`);
+
+promise_test(async t => {
+  const pc1 = new RTCPeerConnection();
+  t.add_cleanup(() => pc1.close());
+  const pc2 = new RTCPeerConnection();
+  t.add_cleanup(() => pc2.close());
+  pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
+  pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
+
+  const track = createResizableTrack(120, 60);
+  t.add_cleanup(() => track.stop());
+  const {sender} = pc1.addTransceiver(track);
+
+  const params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownBy = 2.0;
+  params.encodings[0].scaleResolutionDownTo = { maxWidth: 120, maxHeight: 60 };
+  const p = sender.setParameters(params);
+
+  await pc1.setLocalDescription();
+  await pc2.setRemoteDescription(pc1.localDescription);
+  await pc2.setLocalDescription();
+  await pc1.setRemoteDescription(pc2.localDescription);
+
+  await waitForFrameWithResolution(t, pc1, 120, 60);
+}, `setParameters: scaleResolutionDownBy is ignored when ` +
+   `scaleResolutionDownTo is specified`);
+
+promise_test(async t => {
+  const pc1 = new RTCPeerConnection();
+  t.add_cleanup(() => pc1.close());
+  const pc2 = new RTCPeerConnection();
+  t.add_cleanup(() => pc2.close());
+  pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
+  pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
+
+  const track = createResizableTrack(120, 60);
+  t.add_cleanup(() => track.stop());
+  const {sender} = pc1.addTransceiver(track, {
+      sendEncodings: [{
+          scaleResolutionDownTo: { maxWidth: 60, maxHeight: 30 }
+      }]
+  });
+
+  await pc1.setLocalDescription();
+  await pc2.setRemoteDescription(pc1.localDescription);
+  await pc2.setLocalDescription();
+  await pc1.setRemoteDescription(pc2.localDescription);
+
+  await waitForFrameWithResolution(t, pc1, 60, 30);
+}, `addTransceiver: scaleResolutionDownTo with half resolution`);
+
+promise_test(async t => {
+  const pc1 = new RTCPeerConnection();
+  t.add_cleanup(() => pc1.close());
+  const pc2 = new RTCPeerConnection();
+  t.add_cleanup(() => pc2.close());
+  pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
+  pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
+
+  const track = createResizableTrack(120, 60);
+  t.add_cleanup(() => track.stop());
+  const {sender} = pc1.addTransceiver(track);
+
+  await pc1.setLocalDescription();
+  await pc2.setRemoteDescription(pc1.localDescription);
+  await pc2.setLocalDescription();
+  await pc1.setRemoteDescription(pc2.localDescription);
+
+  // Request full resolution.
+  let params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownTo = { maxWidth: 120, maxHeight: 60 };
+  await sender.setParameters(params);
+  await waitForFrameWithResolution(t, pc1, 120, 60);
+
+  // Request half resolution.
+  params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownTo = { maxWidth: 60, maxHeight: 30 };
+  await sender.setParameters(params);
+  await waitForFrameWithResolution(t, pc1, 60, 30);
+
+  // Request full resolution again.
+  params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownTo = { maxWidth: 120, maxHeight: 60 };
+  await sender.setParameters(params);
+  await waitForFrameWithResolution(t, pc1, 120, 60);
+}, `setParameters: Modify scaleResolutionDownTo while sending`);
+
+promise_test(async t => {
+  const pc1 = new RTCPeerConnection();
+  t.add_cleanup(() => pc1.close());
+  const pc2 = new RTCPeerConnection();
+  t.add_cleanup(() => pc2.close());
+  pc1.onicecandidate = e => pc2.addIceCandidate(e.candidate);
+  pc2.onicecandidate = e => pc1.addIceCandidate(e.candidate);
+
+  const track = createResizableTrack(60, 30);
+  t.add_cleanup(() => track.stop());
+  const {sender} = pc1.addTransceiver(track);
+
+  await pc1.setLocalDescription();
+  await pc2.setRemoteDescription(pc1.localDescription);
+  await pc2.setLocalDescription();
+  await pc1.setRemoteDescription(pc2.localDescription);
+
+  // scaleTo is portrait, track is landscape, but no scaling should happen due
+  // to orientation agnosticism.
+  let params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownTo = { maxWidth: 30, maxHeight: 60 };
+  await sender.setParameters(params);
+  await waitForFrameWithResolution(t, pc1, 60, 30);
+
+  // Change orientation of the track: still no downscale, but encoder begins to
+  // produce the new orientation.
+  track.resize(30, 60);
+  await waitForFrameWithResolution(t, pc1, 30, 60);
+
+  // scaleTo in landscape, reducing to half size. Verify track, which is
+  // portrait, is scaled down by 2.
+  params = sender.getParameters();
+  params.encodings[0].scaleResolutionDownTo = { maxWidth: 30, maxHeight: 15 };
+  await sender.setParameters(params);
+  await waitForFrameWithResolution(t, pc1, 15, 30);
+}, `scaleResolutionDownTo is orientation agnostic`);
+</script>