OKAY!!!

shrunk camera to top right, moved timer to the left uhhhhhh a lot of other shit. currently adding a 4th output. resulting to the total being: (l)eft, (r)ight, r(e)st, and s(a)lute. currently in the middle of cleaning up and fixing the ML trainer so it gives me more accurate predictions cause it fucking sucks once i generate the right model, ill make sure that the r(e)st output triggers properly. also the green dots are still there because i need em for debugging. final version will not have dots. probably after that im gonna go home and then iron out the textures tomorrow lol.... but maybe if i have energy ill add the timer thing. maybe a neat UI thing to ensure the person knows to hold the salute?
DrSpaniel · Apr 21, 2024 · 83c8ca8 · 83c8ca8
1 parent e2b8e30
commit 83c8ca8
Show file tree

Hide file tree

Showing 36 changed files with 487 additions and 59 deletions.
diff --git a/projects/Wint24/ML/YMCA/YMCAtrainer/js/script.js b/projects/Wint24/ML/YMCA/YMCAtrainer/js/script.js
@@ -52,7 +52,7 @@ function setup() {
   };
 
   brain = ml5.neuralNetwork(options);
-  //brain.loadData('controls.json', dataReady);
+  //brain.loadData('controls.json', dataReady);   //comment this to collect data. uncomment AND load the model files to train it. 
 }
 
 function dataReady(){
@@ -66,7 +66,7 @@ function dataReady(){
 
 function finished(){
   console.log('model trained!!');
-  brain.save();   //saves meta, model, and weights to download
+  brain.save();   //saves meta, model, and weights to download to then practice in the model file
 }
 
 function draw() {

diff --git a/projects/Wint24/final/MLtrainer/practice/.vscode/settings.json b/projects/Wint24/final/MLtrainer/practice/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "liveServer.settings.port": 5501
+}
diff --git a/projects/Wint24/final/MLtrainer/practice/README.md b/projects/Wint24/final/MLtrainer/practice/README.md
@@ -0,0 +1,3 @@
+# Template p5 project
+
+This is the README file for the entire project. For more official projects you should write information here about the nature of the project, your name, any special explanations of how the project works, etc.
diff --git a/projects/Wint24/final/MLtrainer/practice/assets/images/bg.jpg b/projects/Wint24/final/MLtrainer/practice/assets/images/bg.jpg
diff --git a/projects/Wint24/final/MLtrainer/practice/assets/images/clown.png b/projects/Wint24/final/MLtrainer/practice/assets/images/clown.png
diff --git a/...t24/final/shipGame/assets/sounds/bark.wav → ...MLtrainer/practice/assets/sounds/bark.wav b/...t24/final/shipGame/assets/sounds/bark.wav → ...MLtrainer/practice/assets/sounds/bark.wav
diff --git a/projects/Wint24/final/MLtrainer/practice/css/style.css b/projects/Wint24/final/MLtrainer/practice/css/style.css
@@ -0,0 +1,20 @@
+/*********************************************
+
+Here is a description of any special CSS used.
+In this case it's just padding and margin to 0
+and then centring the canvas (and anything else)
+using the CSS Grid.
+
+Also hides overflow.
+
+**********************************************/
+
+body {
+  padding: 0;
+  margin: 0;
+  display: grid; /* Set up the grid and justify/align to center */
+  justify-content: center;
+  align-content: center;
+  height: 100vh; /* Body is the height of the viewport */
+  overflow: hidden; /* Hide anything that goes off the window size */
+}
diff --git a/projects/Wint24/final/MLtrainer/practice/index.html b/projects/Wint24/final/MLtrainer/practice/index.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0>
+
+    <title>CHANGE THIS TITLE IN INDEX.HTML!</title>
+
+    <!-- CSS stylesheet(s) -->
+    <link rel="stylesheet" type="text/css" href="css/style.css" />
+
+    <!-- Library script(s) -->
+    <script src="js/libraries/p5.min.js"></script>
+    <script src="js/libraries/p5.sound.min.js"></script>
+    <script src="https://unpkg.com/ml5@latest/dist/ml5.min.js"></script> <!--needs internet access-->
+    <!-- My script(s) -->
+    <script src="js/script.js"></script>
+  </head>
+
+  <body>
+    <!-- HTML would go here if needed. -->
+  </body>
+
+</html>
diff --git a/projects/Wint24/final/MLtrainer/practice/js/libraries/p5.min.js b/projects/Wint24/final/MLtrainer/practice/js/libraries/p5.min.js
diff --git a/projects/Wint24/final/MLtrainer/practice/js/libraries/p5.sound.min.js b/projects/Wint24/final/MLtrainer/practice/js/libraries/p5.sound.min.js
diff --git a/projects/Wint24/final/MLtrainer/practice/js/script.js b/projects/Wint24/final/MLtrainer/practice/js/script.js
@@ -0,0 +1,143 @@
+/**
+ * Daniel Gonzalez
+ *
+ * poseML:
+ * using ml5's pose function, the user will do certain poses
+ * to control a spaceship in the screen. first ill play with pose thing to see what can be done.
+ *
+ */
+"use strict";
+
+let video;
+let poseNet;
+let pose;
+let skeleton;
+
+let brain;
+let poseLabel = "?";
+
+let state = "waiting";
+let targetLabel;
+
+function keyPressed() {
+  if (key == "s") {
+    brain.saveData();
+  } else {
+    targetLabel = key;
+    console.log(targetLabel);
+    setTimeout(function () {
+      console.log("collecting");
+      state = "collecting";
+
+      setTimeout(function () {
+        console.log("not collecting!");
+        state = "waiting";
+      }, 10000);
+    }, 5000);
+  }
+}
+
+function setup() {
+  createCanvas(640, 480);
+  video = createCapture(VIDEO);
+  video.hide(); //hiding, using image video on draw to show video)
+
+  poseNet = ml5.poseNet(video, modelLoaded); //passes through video, and triggers modelloaded when model loads
+  poseNet.on("pose", gotPoses);
+
+  let options = {
+    inputs: 34,   //i wonder if this can be reduced........
+    outputs: 4,   //once adding the new ML, there will be 4 outputs so change this plz
+    task: "classification",
+    debug: true,
+  };
+
+  brain = ml5.neuralNetwork(options);
+
+  const modelInfo = {
+    model: "model/model.json",
+    metadata: "model/model_meta.json",
+    weights: "model/model.weights.bin",
+  };
+  brain.load(modelInfo, brainLoaded);
+}
+
+function brainLoaded() {
+  console.log("pose classification ready!!!!");
+  classifyPose();
+}
+
+function classifyPose() {
+  if (pose) {
+    let inputs = [];
+    for (let i = 0; i < pose.keypoints.length; i++) {
+      let x = pose.keypoints[i].position.x;
+      let y = pose.keypoints[i].position.y;
+      inputs.push(x);
+      inputs.push(y);
+    }
+    brain.classify(inputs, gotResult); //inputs is the 32 inputs!
+  } else {
+    setTimeout(classifyPose, 100);
+  }
+}
+
+function gotResult(error, results) {
+  if (results[0].confidence > 0.75) {
+    poseLabel = results[0].label.toUpperCase();   //you dont even need to type the letter out. the model already has the classifications of each.
+  }
+  console.log(results[0].confidence);   
+  classifyPose();
+}
+
+
+
+
+
+function draw() {
+  push();
+  translate(video.width, 0); //this and line below simply flips the video
+  scale(-1, 1);
+  image(video, 0, 0, video.width, video.height);
+
+  if (pose) {
+
+    for (let i = 0; i < skeleton.length; i++) {
+      let a = skeleton[i][0];
+      let b = skeleton[i][1];
+      strokeWeight(2);
+      stroke(255);
+      line(a.position.x, a.position.y, b.position.x, b.position.y);
+    }
+
+    for (let i = 0; i < pose.keypoints.length; i++) {
+      let x = pose.keypoints[i].position.x;
+      let y = pose.keypoints[i].position.y;
+      fill(0, 255, 0);
+      ellipse(x, y, 16, 16);
+    }
+  }
+  pop();
+  fill(255, 0, 255);
+  noStroke();
+  textSize(256);
+  textAlign(CENTER, CENTER);
+  text(poseLabel, width / 2, height / 2);
+}
+
+function modelLoaded() {
+  console.log("poseNet ready!");
+}
+
+function gotPoses(poses) {
+  //console.log(poses);
+  if (poses.length > 0) {
+    pose = poses[0].pose;
+    skeleton = poses[0].skeleton;
+
+    if (state == "collecting") {
+      let target = [targetLabel];
+      brain.addData(inputs, target);
+    }
+  }
+}
diff --git a/projects/Wint24/final/MLtrainer/practice/model/model.json b/projects/Wint24/final/MLtrainer/practice/model/model.json
@@ -0,0 +1 @@
+{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true,"batch_input_shape":[null,34],"dtype":"float32"}},{"class_name":"Dense","config":{"units":4,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 1.7.4","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./model.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[34,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,4],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[4],"dtype":"float32"}]}]}
diff --git a/projects/Wint24/final/MLtrainer/practice/model/model.weights.bin b/projects/Wint24/final/MLtrainer/practice/model/model.weights.bin
diff --git a/projects/Wint24/final/MLtrainer/practice/model/model_meta.json b/projects/Wint24/final/MLtrainer/practice/model/model_meta.json
@@ -0,0 +1 @@
+{"inputUnits":[34],"outputUnits":4,"inputs":{"0":{"dtype":"number","min":280.3004986952251,"max":383.790803308153},"1":{"dtype":"number","min":147.24558314460725,"max":177.01316981927897},"2":{"dtype":"number","min":296.3618985780946,"max":405.4482396938458},"3":{"dtype":"number","min":134.23344712313047,"max":170.17375664024502},"4":{"dtype":"number","min":262.9262058836941,"max":371.5705806272039},"5":{"dtype":"number","min":136.67460994497813,"max":160.23195904980375},"6":{"dtype":"number","min":332.20154521066394,"max":424.58726934885703},"7":{"dtype":"number","min":139.18400627165917,"max":191.991235755297},"8":{"dtype":"number","min":242.37788129873314,"max":342.4481604906372},"9":{"dtype":"number","min":147.3066491746717,"max":177.198087343446},"10":{"dtype":"number","min":374.87653799094113,"max":417.8781923524137},"11":{"dtype":"number","min":205.67276179094722,"max":273.1209353435828},"12":{"dtype":"number","min":231.7782248411661,"max":276.0204279561915},"13":{"dtype":"number","min":198.86537982332106,"max":277.2764234134659},"14":{"dtype":"number","min":428.9987996023453,"max":498.3865107079888},"15":{"dtype":"number","min":148.8391659510275,"max":374.38947981897496},"16":{"dtype":"number","min":129.73522023004315,"max":224.42286153711697},"17":{"dtype":"number","min":144.1773007166525,"max":383.2367299595696},"18":{"dtype":"number","min":395.12669425994045,"max":605.8538402750335},"19":{"dtype":"number","min":67.91473626162755,"max":492.47824346045115},"20":{"dtype":"number","min":-8.80497973252827,"max":239.47385565316168},"21":{"dtype":"number","min":33.67893738505441,"max":486.7701590571422},"22":{"dtype":"number","min":364.59829248806847,"max":401.50357940317593},"23":{"dtype":"number","min":442.9156494140625,"max":471.7824878024684},"24":{"dtype":"number","min":255.8030184140929,"max":287.2493355005168},"25":{"dtype":"number","min":439.1751514241853,"max":496.6100979222398},"26":{"dtype":"number","min":351.2473864128618,"max":493.4096245079189},"27":{"dtype":"number","min":559.6683846559042,"max":593.8290155908013},"28":{"dtype":"number","min":128.72784210086334,"max":337.79397927369587},"29":{"dtype":"number","min":509.3538728112841,"max":595.3514366298334},"30":{"dtype":"number","min":355.641247344852,"max":472.5765741845513},"31":{"dtype":"number","min":540.8269681819219,"max":572.5829163785111},"32":{"dtype":"number","min":103.02534407678745,"max":371.70568681412635},"33":{"dtype":"number","min":512.3644336744969,"max":572.6645241822714}},"outputs":{"0":{"dtype":"string","min":0,"max":1,"uniqueValues":["e","l","r","a"],"legend":{"e":[1,0,0,0],"l":[0,1,0,0],"r":[0,0,1,0],"a":[0,0,0,1]}}},"isNormalized":true}
diff --git a/projects/Wint24/final/MLtrainer/train/.vscode/settings.json b/projects/Wint24/final/MLtrainer/train/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "liveServer.settings.port": 5501
+}
diff --git a/projects/Wint24/final/MLtrainer/train/README.md b/projects/Wint24/final/MLtrainer/train/README.md
@@ -0,0 +1,3 @@
+# Template p5 project
+
+This is the README file for the entire project. For more official projects you should write information here about the nature of the project, your name, any special explanations of how the project works, etc.
diff --git a/projects/Wint24/final/MLtrainer/train/assets/images/bg.jpg b/projects/Wint24/final/MLtrainer/train/assets/images/bg.jpg
diff --git a/projects/Wint24/final/MLtrainer/train/assets/images/clown.png b/projects/Wint24/final/MLtrainer/train/assets/images/clown.png
diff --git a/projects/Wint24/final/MLtrainer/train/assets/sounds/bark.wav b/projects/Wint24/final/MLtrainer/train/assets/sounds/bark.wav
diff --git a/projects/Wint24/final/MLtrainer/train/controls.json b/projects/Wint24/final/MLtrainer/train/controls.json
diff --git a/projects/Wint24/final/MLtrainer/train/css/style.css b/projects/Wint24/final/MLtrainer/train/css/style.css
@@ -0,0 +1,20 @@
+/*********************************************
+
+Here is a description of any special CSS used.
+In this case it's just padding and margin to 0
+and then centring the canvas (and anything else)
+using the CSS Grid.
+
+Also hides overflow.
+
+**********************************************/
+
+body {
+  padding: 0;
+  margin: 0;
+  display: grid; /* Set up the grid and justify/align to center */
+  justify-content: center;
+  align-content: center;
+  height: 100vh; /* Body is the height of the viewport */
+  overflow: hidden; /* Hide anything that goes off the window size */
+}
diff --git a/projects/Wint24/final/MLtrainer/train/index.html b/projects/Wint24/final/MLtrainer/train/index.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0>
+
+    <title>CHANGE THIS TITLE IN INDEX.HTML!</title>
+
+    <!-- CSS stylesheet(s) -->
+    <link rel="stylesheet" type="text/css" href="css/style.css" />
+
+    <!-- Library script(s) -->
+    <script src="js/libraries/p5.min.js"></script>
+    <script src="js/libraries/p5.sound.min.js"></script>
+    <script src="https://unpkg.com/ml5@latest/dist/ml5.min.js"></script> <!--needs internet access-->
+    <!-- My script(s) -->
+    <script src="js/script.js"></script>
+  </head>
+
+  <body>
+    <!-- HTML would go here if needed. -->
+  </body>
+
+</html>
diff --git a/projects/Wint24/final/MLtrainer/train/js/libraries/p5.min.js b/projects/Wint24/final/MLtrainer/train/js/libraries/p5.min.js
diff --git a/projects/Wint24/final/MLtrainer/train/js/libraries/p5.sound.min.js b/projects/Wint24/final/MLtrainer/train/js/libraries/p5.sound.min.js
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Template p5 project

		This is the README file for the entire project. For more official projects you should write information here about the nature of the project, your name, any special explanations of how the project works, etc.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Dense","config":{"units":16,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true,"batch_input_shape":[null,34],"dtype":"float32"}},{"class_name":"Dense","config":{"units":4,"activation":"softmax","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}}]},"keras_version":"tfjs-layers 1.7.4","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["./model.weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[34,16],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[16],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[16,4],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[4],"dtype":"float32"}]}]}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"inputUnits":[34],"outputUnits":4,"inputs":{"0":{"dtype":"number","min":280.3004986952251,"max":383.790803308153},"1":{"dtype":"number","min":147.24558314460725,"max":177.01316981927897},"2":{"dtype":"number","min":296.3618985780946,"max":405.4482396938458},"3":{"dtype":"number","min":134.23344712313047,"max":170.17375664024502},"4":{"dtype":"number","min":262.9262058836941,"max":371.5705806272039},"5":{"dtype":"number","min":136.67460994497813,"max":160.23195904980375},"6":{"dtype":"number","min":332.20154521066394,"max":424.58726934885703},"7":{"dtype":"number","min":139.18400627165917,"max":191.991235755297},"8":{"dtype":"number","min":242.37788129873314,"max":342.4481604906372},"9":{"dtype":"number","min":147.3066491746717,"max":177.198087343446},"10":{"dtype":"number","min":374.87653799094113,"max":417.8781923524137},"11":{"dtype":"number","min":205.67276179094722,"max":273.1209353435828},"12":{"dtype":"number","min":231.7782248411661,"max":276.0204279561915},"13":{"dtype":"number","min":198.86537982332106,"max":277.2764234134659},"14":{"dtype":"number","min":428.9987996023453,"max":498.3865107079888},"15":{"dtype":"number","min":148.8391659510275,"max":374.38947981897496},"16":{"dtype":"number","min":129.73522023004315,"max":224.42286153711697},"17":{"dtype":"number","min":144.1773007166525,"max":383.2367299595696},"18":{"dtype":"number","min":395.12669425994045,"max":605.8538402750335},"19":{"dtype":"number","min":67.91473626162755,"max":492.47824346045115},"20":{"dtype":"number","min":-8.80497973252827,"max":239.47385565316168},"21":{"dtype":"number","min":33.67893738505441,"max":486.7701590571422},"22":{"dtype":"number","min":364.59829248806847,"max":401.50357940317593},"23":{"dtype":"number","min":442.9156494140625,"max":471.7824878024684},"24":{"dtype":"number","min":255.8030184140929,"max":287.2493355005168},"25":{"dtype":"number","min":439.1751514241853,"max":496.6100979222398},"26":{"dtype":"number","min":351.2473864128618,"max":493.4096245079189},"27":{"dtype":"number","min":559.6683846559042,"max":593.8290155908013},"28":{"dtype":"number","min":128.72784210086334,"max":337.79397927369587},"29":{"dtype":"number","min":509.3538728112841,"max":595.3514366298334},"30":{"dtype":"number","min":355.641247344852,"max":472.5765741845513},"31":{"dtype":"number","min":540.8269681819219,"max":572.5829163785111},"32":{"dtype":"number","min":103.02534407678745,"max":371.70568681412635},"33":{"dtype":"number","min":512.3644336744969,"max":572.6645241822714}},"outputs":{"0":{"dtype":"string","min":0,"max":1,"uniqueValues":["e","l","r","a"],"legend":{"e":[1,0,0,0],"l":[0,1,0,0],"r":[0,0,1,0],"a":[0,0,0,1]}}},"isNormalized":true}