Merge branch 'AlmasB:dev' into dev

fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/WebAPI.java

@@ -45,6 +45,7 @@ private static Map<String, URL> extractURLs() {
                 "tts/index.html",
                 "tts/script.js",
                 "gesturerecog/index.html",
+                "gesturerecog/script.js",
                 "speechrecog/index.html",
                 "speechrecog/script.js"
         ).forEach(relativeURL -> {

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/WebAPIService.kt

@@ -12,12 +12,18 @@ import com.almasb.fxgl.net.ws.LocalWebSocketServer
 import com.almasb.fxgl.net.ws.RPCService
 import javafx.beans.property.ReadOnlyBooleanProperty
 import javafx.beans.property.ReadOnlyBooleanWrapper
+import org.openqa.selenium.JavascriptExecutor
 import org.openqa.selenium.WebDriver
 import org.openqa.selenium.chrome.ChromeDriver
 import org.openqa.selenium.chrome.ChromeOptions
 import org.openqa.selenium.firefox.FirefoxDriver
 import org.openqa.selenium.firefox.FirefoxOptions
+import org.openqa.selenium.logging.LogType
+import org.openqa.selenium.logging.LoggingPreferences
 import java.net.URL
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.logging.Level
 
 /**
  * Provides access to JS-driven implementation.
@@ -30,6 +36,9 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
 
     private val log = Logger.get(WebAPIService::class.java)
 
+    private val logExecutor = Executors.newSingleThreadScheduledExecutor()
+    private var isLoggingScheduled = false
+
     private val readyProp = ReadOnlyBooleanWrapper(false)
 
     var isReady: Boolean
@@ -73,6 +82,15 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
                 webDriver = loadWebDriverAndPage(apiURL)
 
                 onWebDriverLoaded(webDriver!!)
+
+                if (!isLoggingScheduled) {
+                    isLoggingScheduled = true
+
+                    logExecutor.scheduleWithFixedDelay({
+                        transferWebDriverLogs()
+                    }, 1L, 1L, TimeUnit.SECONDS)
+                }
+
             } catch (e: Exception) {
                 log.warning("Failed to start web driver.")
                 log.warning("Error data", e)
@@ -111,11 +129,48 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
     private fun loadChromeDriver(): WebDriver {
         val options = ChromeOptions()
         options.addArguments("--headless=new")
+        // for modules
+        options.addArguments("--allow-file-access-from-files")
+        // for webcam, audio input
         options.addArguments("--use-fake-ui-for-media-stream")
 
+        val logPrefs = LoggingPreferences()
+        logPrefs.enable(LogType.BROWSER, Level.ALL)
+
+        options.setCapability("goog:loggingPrefs", logPrefs)
+
         return ChromeDriver(options)
     }
 
+    /**
+     * Get all console logs and reroute them to FXGL logs.
+     */
+    private fun transferWebDriverLogs() {
+        try {
+            webDriver?.let {
+                it.manage()
+                    .logs()
+                    .get(LogType.BROWSER)
+                    .all
+                    .forEach {
+                        log.debug(it.message)
+                    }
+            }
+        } catch (e: Exception) {
+            log.warning("log error", e)
+        }
+    }
+
+    protected fun executeScript(script: String) {
+        try {
+            webDriver?.let {
+                (it as JavascriptExecutor).executeScript(script)
+            }
+        } catch (e: Exception) {
+            log.warning("Failed to execute script", e)
+        }
+    }
+
     /**
      * Stops this service.
      * No-op if it has not started via start() before.
@@ -134,6 +189,7 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
     }
 
     override fun onExit() {
+        logExecutor.shutdownNow()
         stop()
         super.onExit()
     }

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/gesturerecog/Hand.kt

@@ -6,6 +6,7 @@
 
 package com.almasb.fxgl.intelligence.gesturerecog
 
+import javafx.geometry.Point2D
 import javafx.geometry.Point3D
 
 /**
@@ -22,4 +23,10 @@ data class Hand(
 ) {
 
     fun getPoint(landmark: HandLandmark) = points[landmark.ordinal]
+
+    fun getScaledPoint2D(landmark: HandLandmark, scaleWidth: Double, scaleHeight: Double): Point2D {
+        val p = points[landmark.ordinal]
+
+        return Point2D((1.0 - p.x) * scaleWidth, p.y * scaleHeight)
+    }
 }

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/gesturerecog/HandLandmarksView.kt

@@ -0,0 +1,153 @@
+/*
+ * FXGL - JavaFX Game Library. The MIT License (MIT).
+ * Copyright (c) AlmasB ([email protected]).
+ * See LICENSE for details.
+ */
+
+package com.almasb.fxgl.intelligence.gesturerecog
+
+import com.almasb.fxgl.core.concurrent.Async
+import com.almasb.fxgl.core.math.FXGLMath
+import javafx.scene.layout.Pane
+import javafx.scene.paint.Color
+import javafx.scene.shape.Circle
+import javafx.scene.shape.Line
+import javafx.util.Duration
+import java.util.function.Consumer
+import kotlin.math.max
+import kotlin.math.min
+
+/**
+ * @author Almas Baim (https://github.com/AlmasB)
+ */
+class HandLandmarksView : Pane(), Consumer<Hand> {
+
+    /**
+     * Allows the user configure how the hands are to be visualised.
+     */
+    val config = HandViewConfig(
+        scaleX = 600.0,
+        scaleY = 400.0,
+        scaleZ = 120.0,
+        minRadius = 3.0,
+        maxRadius = 7.0,
+        keepVisibleDuration = Duration.seconds(0.4)
+    )
+
+    // our backend supports exactly 2 hands
+    private val handView1 = HandView()
+    private val handView2 = HandView()
+
+    init {
+        children += handView1
+        children += handView2
+    }
+
+    override fun accept(hand: Hand) {
+        Async.startAsyncFX {
+            if (hand.id == 0) {
+                handView1.update(hand, config)
+            }
+
+            if (hand.id == 1) {
+                handView2.update(hand, config)
+            }
+
+            val now = System.currentTimeMillis()
+
+            // TODO: this isn't quite right because accept() is only called when there is data available
+            // so isVisible = false is never needed
+            if (now - handView1.lastTimeVisibleMillis > config.keepVisibleDuration.toMillis()) {
+                handView1.isVisible = false
+            }
+
+            if (now - handView2.lastTimeVisibleMillis >= config.keepVisibleDuration.toMillis()) {
+                handView2.isVisible = false
+            }
+        }
+    }
+}
+
+data class HandViewConfig(
+    var scaleX: Double,
+    var scaleY: Double,
+    var scaleZ: Double,
+    var minRadius: Double,
+    var maxRadius: Double,
+    var keepVisibleDuration: Duration
+)
+
+private class HandView : Pane() {
+
+    var lastTimeVisibleMillis = 0L
+
+    // nodes
+    private val landmarks = Array(21) { Circle(5.0, 5.0, 5.0, Color.RED) }
+    // edges
+    private val connections = Array(21) { Line() }
+
+    // from https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker
+    private val connectionPairs = listOf(
+        0 to 1,
+        1 to 2,
+        2 to 3,
+        3 to 4,
+        0 to 5,
+        5 to 6,
+        6 to 7,
+        7 to 8,
+        5 to 9,
+        9 to 10,
+        10 to 11,
+        11 to 12,
+        9 to 13,
+        13 to 14,
+        14 to 15,
+        15 to 16,
+        13 to 17,
+        0 to 17,
+        17 to 18,
+        18 to 19,
+        19 to 20
+    )
+
+    init {
+        connections.forEach { it.stroke = Color.GREEN }
+
+        children.addAll(connections)
+        children.addAll(landmarks)
+
+        isVisible = false
+    }
+
+    fun update(hand: Hand, config: HandViewConfig) {
+        hand.points.forEachIndexed { i, p ->
+            landmarks[i].translateX = (1.0 - p.x) * config.scaleX
+            landmarks[i].translateY = p.y * config.scaleY
+
+            var radius = FXGLMath.abs(p.z) * config.scaleZ
+            radius = max(radius, config.minRadius)
+            radius = min(radius, config.maxRadius)
+
+            landmarks[i].centerX = radius
+            landmarks[i].centerY = radius
+            landmarks[i].radius = radius
+        }
+
+        connectionPairs.forEachIndexed { i, pair ->
+            val p1 = landmarks[pair.first]
+            val p2 = landmarks[pair.second]
+
+            val r1 = landmarks[pair.first].radius
+            val r2 = landmarks[pair.second].radius
+
+            connections[i].startX = p1.translateX + r1
+            connections[i].startY = p1.translateY + r1
+            connections[i].endX = p2.translateX + r2
+            connections[i].endY = p2.translateY + r2
+        }
+
+        isVisible = true
+        lastTimeVisibleMillis = System.currentTimeMillis()
+    }
+}