Import of Selkies JS Interposer for WebRTC to Linux Joystick support

- LD_PRELOAD syscall interposer to emulate gamepads. - Intercepts open() requests to /dev/input/jsX devices. - Interposer connects to selkies unix domain socket. - Interposer writes log to /tmp/selkies_js.log - Selkies WebRTC app sends Javascript gamepad events to socket. - Up to 4 simultaneous gamepads supported from WebRTC. - Build as debian package and installed to /usr/local/lib/selkies-js-interposer/joystick_interposer.so - Dev container updated to auto-load interposer. - Update to github workflow to build and publish image and release asset. - Add joystick interposer to example Dockerfile
selkies-project · Sep 22, 2023 · be3c2f0 · be3c2f0
1 parent 1a4bebd
commit be3c2f0
Show file tree

Hide file tree

Showing 27 changed files with 1,653 additions and 201 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -71,3 +71,8 @@ RUN cd /tmp && sudo apt-get update && \
     sudo DEBIAN_FRONTEND=noninteractive apt-get install -y xdg-utils ./google-chrome-stable_current_amd64.deb && \
     sudo rm -f google-chrome-stable_current_amd64.deb && \
     xdg-settings set default-web-browser google-chrome.desktop
+
+# Install other dev utils
+RUN sudo apt-get update && sudo apt-get install -y \
+    jstest-gtk \
+    netcat
diff --git a/.devcontainer/features/desktop-selkies/src/start-selkies.sh b/.devcontainer/features/desktop-selkies/src/start-selkies.sh
@@ -37,6 +37,16 @@ export PULSE_SERVER=tcp:127.0.0.1:4713
 sudo /usr/bin/pulseaudio -k >/dev/null 2>&1
 sudo /usr/bin/pulseaudio --daemonize --system --verbose --log-target=file:/tmp/pulseaudio.log --realtime=true --disallow-exit -L 'module-native-protocol-tcp auth-ip-acl=127.0.0.0/8 port=4713 auth-anonymous=1'
 
+# Create /dev/input/jsX if they don't already exists
+sudo mkdir -p /dev/input
+sudo touch /dev/input/{js0,js1,js2,js3}
+
+# If installed, add the joystick interposer to the LD_PRELOAD environment
+if [[ -e /usr/local/lib/selkies-js-interposer/joystick_interposer.so ]]; then
+    export LD_PRELOAD=${LD_PRELOAD}:/usr/local/lib/selkies-js-interposer/joystick_interposer.so
+    export SDL_JOYSTICK_DEVICE=/dev/input/js0
+fi
+
 # Start desktop environment
 case ${DESKTOP:-XFCE} in
     FLUXBOX)

diff --git a/.github/workflows/build_and_publish_all_images.yaml b/.github/workflows/build_and_publish_all_images.yaml
@@ -38,6 +38,18 @@ jobs:
             version_suffix: -ubuntu22.04
             build_args: UBUNTU_RELEASE=22.04
             source_directory: addons/gstreamer
+
+          - name: js-interposer
+            version_suffix: -ubuntu20.04
+            build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
+            source_directory: addons/js-interposer
+            dockerfile: Dockerfile.ubuntu_debpkg
+
+          - name: js-interposer
+            version_suffix: -ubuntu22.04
+            build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
+            source_directory: addons/js-interposer
+            dockerfile: Dockerfile.ubuntu_debpkg
 
           - name: infra-gcp-installer
             source_directory: infra/gce/installer-image
@@ -59,6 +71,7 @@ jobs:
           image_name: ${{ matrix.name }}
           image_source_directory: ${{ matrix.source_directory }}
           image_version_1: $GITHUB_REF_NAME${{ matrix.version_suffix }}
+          dockerfile: ${{ matrix.dockerfile || 'Dockerfile' }}
 
   # Note: When modifying this job, copy modifications to all other workflows' image jobs.
   all_example_images:
@@ -69,13 +82,13 @@ jobs:
         include:
           - name: gst-py-example
             version_suffix: -ubuntu20.04
-            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }}
+            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}
             dockerfile: Dockerfile.example
             source_directory: .
 
           - name: gst-py-example
             version_suffix: -ubuntu22.04
-            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }}
+            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}
             dockerfile: Dockerfile.example
             source_directory: .
 

diff --git a/.github/workflows/build_changed_images.yaml b/.github/workflows/build_changed_images.yaml
@@ -46,6 +46,16 @@ jobs:
             build_args: UBUNTU_RELEASE=22.04
             source_directory: addons/gstreamer
             source_files_for_diff: addons/gstreamer
+
+          - name: js-interposer
+            version_suffix: -ubuntu20.04
+            build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
+            source_directory: addons/js-interposer
+
+          - name: js-interposer
+            version_suffix: -ubuntu22.04
+            build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
+            source_directory: addons/js-interposer
 
           - name: infra-gcp-installer
             source_directory: infra/gce/installer-image
@@ -95,7 +105,7 @@ jobs:
           - name: gst-py-example
             version_suffix: -ubuntu20.04
             push_image: "false"
-            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=pr;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr
+            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=pr${{ matrix.version_suffix }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr${{ matrix.version_suffix }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr${{ matrix.version_suffix }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:pr${{ matrix.version_suffix }}
             dockerfile: Dockerfile.example
             source_directory: .
             source_files_for_diff: |
@@ -110,7 +120,7 @@ jobs:
           - name: gst-py-example
             version_suffix: -ubuntu22.04
             push_image: "false"
-            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=pr;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr
+            build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=pr${{ matrix.version_suffix }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr${{ matrix.version_suffix }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr${{ matrix.version_suffix }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:pr${{ matrix.version_suffix }}
             dockerfile: Dockerfile.example
             source_directory: .
             source_files_for_diff: |

diff --git a/.github/workflows/publish_release.yaml b/.github/workflows/publish_release.yaml
@@ -52,6 +52,16 @@ jobs:
             version_suffix: -ubuntu22.04
             build_args: UBUNTU_RELEASE=22.04
             source_directory: addons/gstreamer
+
+          - name: js-interposer
+            version_suffix: -ubuntu20.04
+            build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=${{ needs.get_semver.outputs.semver }}
+            source_directory: addons/js-interposer
+
+          - name: js-interposer
+            version_suffix: -ubuntu22.04
+            build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=${{ needs.get_semver.outputs.semver }}
+            source_directory: addons/js-interposer
 
           - name: infra-gcp-installer
             source_directory: infra/gce/installer-image
@@ -86,13 +96,13 @@ jobs:
         include:
           - name: gst-py-example
             version_suffix: -ubuntu20.04
-            build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME
+            build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME;JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:$GITHUB_REF_NAME
             dockerfile: Dockerfile.example
             source_directory: .
 
           - name: gst-py-example
             version_suffix: -ubuntu22.04
-            build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME
+            build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME;JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:$GITHUB_REF_NAME
             dockerfile: Dockerfile.example
             source_directory: .
 
@@ -126,6 +136,14 @@ jobs:
       gst22_mimetype: ${{ steps.extract.outputs.gst22_mimetype }}
       gst22_name: ${{ steps.extract.outputs.gst22_name }}
       gst22_path: ${{ steps.extract.outputs.gst22_path }}
+      js20_cache_key: ${{ steps.extract.outputs.js20_cache_key }}
+      js20_mimetype: ${{ steps.extract.outputs.js20_mimetype }}
+      js20_name: ${{ steps.extract.outputs.js20_name }}
+      js20_path: ${{ steps.extract.outputs.js20_path }}
+      js22_cache_key: ${{ steps.extract.outputs.js22_cache_key }}
+      js22_mimetype: ${{ steps.extract.outputs.js22_mimetype }}
+      js22_name: ${{ steps.extract.outputs.js22_name }}
+      js22_path: ${{ steps.extract.outputs.js22_path }}
       py_cache_key: ${{ steps.extract.outputs.py_cache_key }}
       py_mimetype: ${{ steps.extract.outputs.py_mimetype }}
       py_name: ${{ steps.extract.outputs.py_name }}
@@ -156,6 +174,26 @@ jobs:
             target_directory: /tmp
             target_name: selkies-gstreamer-${{ github.ref_name }}-ubuntu22.04.tgz
             upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/
+
+          - id: js20
+            cache_key: js-interposer-asset-ubuntu2004
+            description: JS Interposr Ubuntu 20.04
+            image_tag: ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}-ubuntu20.04
+            mimetype: application/octet-stream
+            source_path: /opt/selkies-js-interposer_${{ needs.get_semver.outputs.semver }}.deb
+            target_directory: /tmp
+            target_name: selkies-js-interposer-${{ github.ref_name }}-ubuntu20.04.deb
+            upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/
+
+          - id: js22
+            cache_key: js-interposer-asset-ubuntu2204
+            description: JS Interposr Ubuntu 22.04
+            image_tag: ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}-ubuntu22.04
+            mimetype: application/octet-stream
+            source_path: /opt/selkies-js-interposer_${{ needs.get_semver.outputs.semver }}.deb
+            target_directory: /tmp
+            target_name: selkies-js-interposer-${{ github.ref_name }}-ubuntu22.04.deb
+            upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/
 
           - id: py
             cache_key: gst-py-asset
@@ -226,6 +264,18 @@ jobs:
           key: ${{ needs.all_assets.outputs.gst20_cache_key }}
           path: ${{ needs.all_assets.outputs.gst20_path }}
 
+      - name: Ubuntu 22.04 cache read
+        uses: actions/cache@v3
+        with:
+          key: ${{ needs.all_assets.outputs.gst22_cache_key }}
+          path: ${{ needs.all_assets.outputs.gst22_path }}
+
+      - name: JS Interposer Ubuntu 20.04 cache read
+        uses: actions/cache@v3
+        with:
+          key: ${{ needs.all_assets.outputs.js20_cache_key }}
+          path: ${{ needs.all_assets.outputs.js20_path }}
+
       - name: Ubuntu 22.04 cache read
         uses: actions/cache@v3
         with:
@@ -261,6 +311,24 @@ jobs:
           file: ${{ needs.all_assets.outputs.gst22_path }}
           asset_name: ${{ needs.all_assets.outputs.gst22_name }}
           overwrite: true
+
+      - name: JS Interceptor Ubuntu 20.04 upload
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref }}
+          file: ${{ needs.all_assets.outputs.js20_path }}
+          asset_name: ${{ needs.all_assets.outputs.js20_name }}
+          overwrite: true
+
+      - name: JS Interceptor Ubuntu 22.04 upload
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref }}
+          file: ${{ needs.all_assets.outputs.js22_path }}
+          asset_name: ${{ needs.all_assets.outputs.js22_name }}
+          overwrite: true
 
       - name: Python upload
         uses: svenstaro/upload-release-action@v2

diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,5 @@ addons/_archive
 dist
 coturn_env
 src/*.egg-info
-NOTES.md
+NOTES.md
+**/__pycache__/
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
@@ -0,0 +1,24 @@
+{
+    "env": {
+        "myDefaultIncludePath": [
+            "/usr/include/**",
+            "${workspaceFolder}/**",
+        ]
+    },
+    "configurations": [
+        {
+            "name": "Dev",
+            "includePath": [
+                "${myDefaultIncludePath}"
+            ],
+            "browse": {
+                "path": [
+                    "${myDefaultIncludePath}"
+                ],
+                "limitSymbolsToIncludedHeaders": true,
+                "databaseFilename": ""
+            }
+        }
+    ],
+    "version": 4
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,6 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.python"
+    },
+    "python.formatting.provider": "none"
+}
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -3,6 +3,15 @@
     // for the documentation about the tasks.json format
     "version": "2.0.0",
     "tasks": [
+        {
+            "label": "[build] build joystick interposer library",
+            "type": "shell",
+            "command": "sudo make install",
+            "problemMatcher": [],
+            "options": {
+                "cwd": "${workspaceFolder}/addons/js-interposer"
+            }
+        },
         {
             "label": "[build] build python package",
             "type": "shell",
@@ -34,6 +43,7 @@
             "problemMatcher": [],
             "dependsOrder": "sequence",
             "dependsOn": [
+                "[build] build joystick interposer library",
                 "[build] build python package",
                 "[install] re-install python package",
                 "[run] Start selkies-gstreamer"

diff --git a/Dockerfile.example b/Dockerfile.example
@@ -8,9 +8,11 @@ ARG GSTREAMER_BASE_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gstreamer
 ARG GSTREAMER_BASE_IMAGE_RELEASE=main
 ARG PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:main
 ARG WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:main
+ARG JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:main
 FROM ${GSTREAMER_BASE_IMAGE}:${GSTREAMER_BASE_IMAGE_RELEASE}-ubuntu${UBUNTU_RELEASE} as selkies-gstreamer
 FROM ${PY_BUILD_IMAGE} as selkies-build
 FROM ${WEB_IMAGE} as selkies-web
+FROM ${JS_IMAGE}-ubuntu${UBUNTU_RELEASE} as selkies-js-interposer
 FROM ubuntu:${UBUNTU_RELEASE}
 ARG UBUNTU_RELEASE
 
@@ -120,6 +122,10 @@ COPY --from=selkies-gstreamer /opt/gstreamer ./gstreamer
 # Install web application
 COPY --from=selkies-web /usr/share/nginx/html ./gst-web
 
+# Install Joystick Interposer
+COPY --from=selkies-js-interposer /opt/*.deb /opt/selkies-js-interposer.deb
+RUN apt-get install -y /opt/selkies-js-interposer.deb
+
 # Update PWA manifest.json with application information and route.
 ARG PWA_APP_NAME="Selkies WebRTC"
 ARG PWA_APP_SHORT_NAME="selkies"
@@ -174,6 +180,10 @@ export DISPLAY=:0\n\
 export GST_DEBUG=*:2\n\
 export GSTREAMER_PATH=/opt/gstreamer\n\
 source /opt/gstreamer/gst-env\n\
+export LD_PRELOAD=/usr/local/lib/selkies-js-interposer/joystick_interposer.so\n\
+export SDL_JOYSTICK_DEVICE=/dev/input/js0\n\
+sudo mkdir -p /dev/input\n\
+sudo touch /dev/input/{js0,js1,js2,js3}\n\
 Xvfb -screen :0 8192x4096x24 +extension RANDR +extension GLX +extension MIT-SHM -nolisten tcp -noreset -shmem 2>&1 >/tmp/Xvfb.log &\n\
 until [[ -S /tmp/.X11-unix/X0 ]]; do sleep 1; done && echo 'X Server is ready'\n\
 export PULSE_SERVER=unix:/run/pulse/native\n\

diff --git a/addons/gst-web/src/app.js b/addons/gst-web/src/app.js
@@ -439,12 +439,14 @@ webrtc.onconnectionstatechange = (state) => {
 webrtc.ondatachannelopen = () => {
     // Bind gamepad connected handler.
     webrtc.input.ongamepadconnected = (gamepad_id) => {
+        webrtc._setStatus('Gamepad connected: ' + gamepad_id);
         app.gamepadState = "connected";
         app.gamepadName = gamepad_id;
     }
 
     // Bind gamepad disconnect handler.
     webrtc.input.ongamepaddisconnected = () => {
+        webrtc._setStatus('Gamepad disconnected: ' + gamepad_id);
         app.gamepadState = "disconnected";
         app.gamepadName = "none";
     }