From 00e23b23ab3ef12cfa426a60c22b0dd3b3ae96a0 Mon Sep 17 00:00:00 2001 From: Li Li Date: Fri, 6 Mar 2020 11:31:47 -0500 Subject: [PATCH 1/4] Stopped stream if there are any before setting up camera --- .../mit/appinventor/ai/posenet/assets/app.js | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js b/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js index 398a84291..3fb786cef 100644 --- a/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js +++ b/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js @@ -18,6 +18,7 @@ const ERRORS = { let forwardCamera = true; let running = false; +let stream = null; async function setupCamera() { if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { @@ -29,13 +30,18 @@ async function setupCamera() { const video = document.getElementById('video'); video.width = videoWidth; video.height = videoHeight; - - video.srcObject = await navigator.mediaDevices.getUserMedia({ - 'audio': false, - 'video': { - facingMode: forwardCamera ? 'user' : 'environment' - } + if (stream != null) { + stream.getTracks().forEach(t => { + t.stop(); + }); + } + stream = await navigator.mediaDevices.getUserMedia({ + 'audio': false, + 'video': { + facingMode: forwardCamera ? 'user' : 'environment' + } }); + video.srcObject = stream; return new Promise((resolve) => { video.onloadedmetadata = () => { @@ -106,7 +112,7 @@ async function loadModel() { }); } catch (e) { PosenetExtension.error(ERROR_MODEL_LOAD, - ERRORS[ERROR_MODEL_LOAD]); + ERRORS.ERROR_MODEL_LOAD); throw e; } } From 8eaedce4ca81bd4c6e8e04a8ff85c25933d30b9d Mon Sep 17 00:00:00 2001 From: Li Li Date: Mon, 9 Mar 2020 10:47:54 -0400 Subject: [PATCH 2/4] Made video width and height configurable --- .../appinventor/ai/posenet/PosenetExtension.java | 12 +++++++++++- .../edu/mit/appinventor/ai/posenet/assets/app.js | 13 ++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java b/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java index 5d3394247..4e33465a6 100644 --- a/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java +++ b/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java @@ -52,7 +52,7 @@ import org.json.JSONException; import org.json.JSONObject; -@DesignerComponent(version = 20200304, +@DesignerComponent(version = 20200309, category = ComponentCategory.EXTENSION, description = "An extension that embeds a Posenet model.", iconName = "aiwebres/icon.png", @@ -444,6 +444,16 @@ public String UseCamera() { return cameraMode; } + @SimpleProperty (description = "Configure video width.") + public void SetVideoWidth(int width) { + webview.evaluateJavascript("setVideoWidth(" + width + ");", null); + } + + @SimpleProperty (description = "Configure video height.") + public void SetVideoHeight(int height) { + webview.evaluateJavascript("setVideoHeight(" + height + ");", null); + } + private static void requestHardwareAcceleration(Activity activity) { activity.getWindow().setFlags(LayoutParams.FLAG_HARDWARE_ACCELERATED, LayoutParams.FLAG_HARDWARE_ACCELERATED); diff --git a/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js b/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js index 3fb786cef..67b595219 100644 --- a/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js +++ b/appinventor/components/src/edu/mit/appinventor/ai/posenet/assets/app.js @@ -8,14 +8,13 @@ console.log("Posenet Extension using tfjs-converter version " + tf.version_conve const ERROR_WEBVIEW_NO_MEDIA = 400; const ERROR_MODEL_LOAD = 401; -const videoWidth = 300; -const videoHeight = 250; - const ERRORS = { ERROR_WEBVIEW_NO_MEDIA: "WebView does not support navigator.mediaDevices", ERROR_MODEL_LOAD: "Unable to load model" }; +let videoWidth = 300; +let videoHeight = 250; let forwardCamera = true; let running = false; let stream = null; @@ -158,6 +157,14 @@ function setCameraFacingMode(useForward) { }) } +function setVideoWidth(width) { + videoWidth = width; +} + +function setVideoHeight(height) { + videoHeight = height; +} + // noinspection JSUnresolvedVariable navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; From 53e64fc388e7c4805cc6f83091e063bdc01eabcc Mon Sep 17 00:00:00 2001 From: Li Li Date: Mon, 9 Mar 2020 15:50:15 -0400 Subject: [PATCH 3/4] Add canvas on the extension --- .../ai/posenet/PosenetExtension.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java b/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java index 4e33465a6..f6acaa243 100644 --- a/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java +++ b/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java @@ -28,6 +28,7 @@ import com.google.appinventor.components.common.ComponentCategory; import com.google.appinventor.components.common.PropertyTypeConstants; import com.google.appinventor.components.runtime.AndroidNonvisibleComponent; +import com.google.appinventor.components.runtime.Canvas; import com.google.appinventor.components.runtime.Deleteable; import com.google.appinventor.components.runtime.EventDispatcher; import com.google.appinventor.components.runtime.Form; @@ -73,6 +74,7 @@ public class PosenetExtension extends AndroidNonvisibleComponent private static final String FRONT_CAMERA = "Front"; private WebView webview = null; + private Canvas canvas = null; private final Map keyPoints = new ConcurrentHashMap<>(); private double minPoseConfidence = 0.1; private double minPartConfidence = 0.5; @@ -190,6 +192,16 @@ public void WebViewer(WebViewer webviewer) { } } + @SuppressWarnings("squid:S00100") + @DesignerProperty(editorType = PropertyTypeConstants.PROPERTY_TYPE_COMPONENT + + ":com.google.appinventor.components.runtime.Canvas") + @SimpleProperty(userVisible = false) + public void Canvas(Canvas canvas) { + if (canvas != null) { + this.canvas = canvas; + } + } + public void Initialize() { if (webview != null) { initialized = true; @@ -400,6 +412,10 @@ public boolean Enabled() { @SuppressWarnings("squid:S00100") @SimpleEvent(description = "Event indicating that the classifier is ready.") public void ModelReady() { + if (canvas != null) { + SetVideoHeight(canvas.Height()); + SetVideoWidth(canvas.Width()); + } EventDispatcher.dispatchEvent(this, "ModelReady"); } @@ -419,6 +435,7 @@ public void PoseUpdated() { @SuppressWarnings("squid:S00100") @SimpleEvent(description = "Event indicating that a new video frame is ready. ") public void VideoUpdated() { + canvas.BackgroundImageinBase64(BackgroundImage); EventDispatcher.dispatchEvent(this, "VideoUpdated"); } From 9cea646edc6ef1c3da787eea9dbcbf54fa29c15b Mon Sep 17 00:00:00 2001 From: Li Li Date: Tue, 20 Apr 2021 10:23:03 -0400 Subject: [PATCH 4/4] Address comments to not include SET in property name --- .../mit/appinventor/ai/posenet/PosenetExtension.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java b/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java index f6acaa243..d8b58746f 100644 --- a/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java +++ b/appinventor/components/src/edu/mit/appinventor/ai/posenet/PosenetExtension.java @@ -53,7 +53,7 @@ import org.json.JSONException; import org.json.JSONObject; -@DesignerComponent(version = 20200309, +@DesignerComponent(version = 20210420, category = ComponentCategory.EXTENSION, description = "An extension that embeds a Posenet model.", iconName = "aiwebres/icon.png", @@ -413,8 +413,8 @@ public boolean Enabled() { @SimpleEvent(description = "Event indicating that the classifier is ready.") public void ModelReady() { if (canvas != null) { - SetVideoHeight(canvas.Height()); - SetVideoWidth(canvas.Width()); + VideoHeight(canvas.Height()); + VideoWidth(canvas.Width()); } EventDispatcher.dispatchEvent(this, "ModelReady"); } @@ -462,12 +462,12 @@ public String UseCamera() { } @SimpleProperty (description = "Configure video width.") - public void SetVideoWidth(int width) { + public void VideoWidth(int width) { webview.evaluateJavascript("setVideoWidth(" + width + ");", null); } @SimpleProperty (description = "Configure video height.") - public void SetVideoHeight(int height) { + public void VideoHeight(int height) { webview.evaluateJavascript("setVideoHeight(" + height + ");", null); }