Implement adaptive throttling for mouse events (#269)

Fixes #268

app/main.py

@@ -92,14 +92,15 @@ def socket_mouse_event(message):
         mouse_move_event = mouse_event_request.parse_mouse_event(message)
     except mouse_event_request.Error as e:
         logger.error('Failed to parse mouse event request: %s', e)
-        return
+        return {'success': False}
     try:
         fake_mouse.send_mouse_event(mouse_path, mouse_move_event.buttons,
                                     mouse_move_event.relative_x,
                                     mouse_move_event.relative_y)
     except hid_write.WriteError as e:
         logger.error('Failed to forward mouse event: %s', e)
-        return
+        return {'success': False}
+    return {'success': True}
 
 
 @socketio.on('keyRelease')

app/static/js/app.js

@@ -109,6 +109,30 @@ function isIgnoredKeystroke(keyCode) {
   return isModifierKeyCode(keyCode) && isKeycodeAlreadyPressed(keyCode);
 }
 
+function recalculateMouseEventThrottle(
+  currentThrottle,
+  lastRtt,
+  lastWriteSucceeded
+) {
+  const maxThrottleInMilliseconds = 2000;
+  if (!lastWriteSucceeded) {
+    // Apply a 500 ms penalty to the throttle every time an event fails.
+    return Math.min(currentThrottle + 500, maxThrottleInMilliseconds);
+  }
+  // Assume that the server can process messages in roughly half the round trip
+  // time between an event message and its response.
+  const roughSendTime = lastRtt / 2;
+
+  // Set the new throttle to a weighted average between the last throttle time
+  // and the last send time, with a 2/3 bias toward the last send time.
+  const newThrottle = (roughSendTime * 2 + currentThrottle) / 3;
+  return Math.min(newThrottle, maxThrottleInMilliseconds);
+}
+
+function unixTime() {
+  return new Date().getTime();
+}
+
 function browserLanguage() {
   if (navigator.languages) {
     return navigator.languages[0];
@@ -190,11 +214,25 @@ function sendMouseEvent(buttons, relativeX, relativeY) {
   if (!connectedToServer) {
     return;
   }
-  socket.emit("mouse-event", {
-    buttons,
-    relativeX,
-    relativeY,
-  });
+  const remoteScreen = document.getElementById("remote-screen");
+  const requestStartTime = unixTime();
+  socket.emit(
+    "mouse-event",
+    {
+      buttons,
+      relativeX,
+      relativeY,
+    },
+    (response) => {
+      const requestEndTime = unixTime();
+      const requestRtt = requestEndTime - requestStartTime;
+      remoteScreen.millisecondsBetweenMouseEvents = recalculateMouseEventThrottle(
+        remoteScreen.millisecondsBetweenMouseEvents,
+        requestRtt,
+        response.success
+      );
+    }
+  );
 }
 
 function onKeyUp(evt) {

app/templates/custom-elements/remote-screen.html

@@ -52,7 +52,7 @@
         constructor() {
           super();
           this.onWindowResize = this.onWindowResize.bind(this);
-          // Define this when the _.throttle function is loaded.
+          this.isUnderscoreLibraryLoaded = false;
           this.throttledSendMouseEvent = undefined;
 
           // Prevent drag on screen for Firefox.
@@ -78,15 +78,9 @@
           this.shadowRoot
             .getElementById("underscore-library")
             .addEventListener("load", () => {
-              // To avoid overwhelming the connection to the backend or its
-              // fake HID interface, throttle mouse move events.
-              const eventsPerSecond = 20;
-              this.throttledSendMouseEvent = _.throttle(
-                function (evt) {
-                  this.sendMouseEvent(evt);
-                },
-                /*wait=*/ 1000 / eventsPerSecond,
-                { trailing: false }
+              this.isUnderscoreLibraryLoaded = true;
+              this.throttledSendMouseEvent = this.makeThrottledSendMouseEvent(
+                this.millisecondsBetweenMouseEvents
               );
             });
 
@@ -95,11 +89,8 @@
           screenImg.addEventListener("mousemove", (evt) => {
             // Ensure that mouse drags don't attempt to drag the image on the screen.
             evt.preventDefault();
-            if (this.throttledSendMouseEvent) {
-              this.throttledSendMouseEvent(evt);
-            } else {
-              this.sendMouseEvent(evt);
-            }
+
+            this.throttledSendMouseEvent(evt);
           });
           screenImg.addEventListener("mousedown", this.sendMouseEvent);
           screenImg.addEventListener("mouseup", this.sendMouseEvent);
@@ -125,6 +116,16 @@
           this.setAttribute("fullscreen", newValue);
         }
 
+        get millisecondsBetweenMouseEvents() {
+          return parseInt(
+            this.getAttribute("milliseconds-between-mouse-events")
+          );
+        }
+
+        set millisecondsBetweenMouseEvents(newValue) {
+          this.setAttribute("milliseconds-between-mouse-events", newValue);
+        }
+
         get cursor() {
           return this.shadowRoot
             .querySelector(".screen-wrapper")
@@ -138,14 +139,29 @@
         }
 
         static get observedAttributes() {
-          return ["fullscreen"];
+          return ["fullscreen", "milliseconds-between-mouse-events"];
         }
 
         attributeChangedCallback(name, oldValue, newValue) {
           if (name === "fullscreen" && newValue === "true") {
             this.shadowRoot
               .querySelector(".screen-wrapper")
               .requestFullscreen();
+          } else if (
+            name === "milliseconds-between-mouse-events" &&
+            oldValue !== newValue
+          ) {
+            // Note: This is a bit of a hack. When we replace the throttled
+            // event, it resets the throttle. To *sort of* preserve the
+            // throttle, wait to replace it until 90% of the last throttle time
+            // elapsed.
+            const newThrottle = parseInt(newValue);
+            const oldThrottle = parseInt(oldValue) || newThrottle;
+            setTimeout(() => {
+              this.throttledSendMouseEvent = this.makeThrottledSendMouseEvent(
+                newThrottle
+              );
+            }, Math.min(oldThrottle, newThrottle) * 0.9);
           }
         }
 
@@ -171,6 +187,19 @@
           }
         }
 
+        makeThrottledSendMouseEvent(waitInMilliseconds) {
+          if (!this.isUnderscoreLibraryLoaded) {
+            return this.sendMouseEvent;
+          }
+          return _.throttle(
+            function (evt) {
+              this.sendMouseEvent(evt);
+            },
+            waitInMilliseconds,
+            { trailing: false }
+          );
+        }
+
         sendMouseEvent(evt) {
           const boundingRect = evt.target.getBoundingClientRect();
           const cursorX = Math.max(0, evt.clientX - boundingRect.left);

app/templates/index.html

@@ -32,7 +32,10 @@ <h3 id="error-type">Error Type</h3>
 
         <shutdown-dialog id="shutdown-dialog"></shutdown-dialog>
 
-        <remote-screen id="remote-screen"></remote-screen>
+        <remote-screen
+          id="remote-screen"
+          milliseconds-between-mouse-events="600"
+        ></remote-screen>
 
         {% include "components/keyboard-panel.html" %}
       </div>