Added AR library (#286)

* Added AR module

* Fix missing exit toggle

* Add base64 image

* Added export and import objects to json string

* Fix json parsing for ui

* Clean code and added hit-test controls

* Fix error

* Fix highlight not working

* Add highlight for UI component

* Improve outline

* Fix lighting issue for UI

* Improve parsing of state

* Remove convert to string

* Clean up code

* Clean code

* Added lint ignore for skeletonutils.js

* Remove unrequired width and height variables

* Clean up calibration library

* Improved documentation for controls library

* Object state library clean up

* Added more documentation for object library

* Add syncing of movement

* Address issues brought up in review

* Rename arObject parameters

* Add ability to get the object in front

* Fix line-of-sight selection

* Use saar package

* Remove imports for ar, replace with saar

* Increase saar version

* Fix imports

* Fix imports for tab

* Remove gl

* change entries to values

* Fix spring console spam

---------

Co-authored-by: Yin Joe Ng <[email protected]>

modules.json

@@ -99,6 +99,11 @@
       "physics_2d"
     ]
   },
+  "ar": {
+    "tabs": [
+      "AugmentedReality"
+    ]
+  },
   "communication": {
     "tabs": []
   }

package.json

@@ -116,6 +116,7 @@
     "react-ace": "^10.1.0",
     "react-dom": "^18.2.0",
     "regl": "^2.1.0",
+    "saar": "^1.0.4",
     "save-file": "^2.3.1",
     "source-academy-utils": "^1.0.0",
     "source-academy-wabt": "^1.0.4",
@@ -131,7 +132,6 @@
   },
   "resolutions": {
     "@types/react": "^18.2.0",
-    "esbuild": "^0.18.20",
-    "**/gl": "^6.0.2"
+    "esbuild": "^0.18.20"
   }
 }

src/bundles/ar/AR.ts

@@ -0,0 +1,292 @@
+import { ARObject } from 'saar/libraries/object_state_library/ARObject';
+import { OverlayHelper, Toggle } from './OverlayHelper';
+import { Vector3 } from 'saar/libraries/misc';
+
+export class ARState {
+  arObjects: ARObject[] = [];
+  highlightFrontObject: boolean = false;
+  selectedObjectId: string | undefined = undefined;
+  overlay = new OverlayHelper();
+  clickCallbacks = new Map<string, () => void>();
+  onClickCallback = (id: string) => {
+    const callback = this.clickCallbacks.get(id);
+    callback?.();
+  };
+}
+
+/**
+ * Initialize AR.
+ */
+export function initAR() {
+  (window as any).arController = new ARState();
+}
+
+/**
+ * Obtains AR module state.
+ *
+ * @returns Current AR state.
+ */
+export function getModuleState(): ARState {
+  return (window as any).arController as ARState;
+}
+
+/**
+ * Calls callback to update AR context in tab.
+ */
+export function callARCallback() {
+  const f = (window as any).arControllerCallback as Function;
+  if (f) {
+    f();
+  }
+}
+
+// Overlay
+
+/**
+ * Sets the left toggle.
+ *
+ * @param text Label on toggle.
+ * @param callback Function to call when toggle is clicked.
+ */
+export function setLeftToggle(text: string, callback: () => void) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.overlay.toggleLeft = new Toggle(text, callback);
+}
+
+/**
+ * Sets the center toggle.
+ *
+ * @param text Label on toggle.
+ * @param callback Function to call when toggle is clicked.
+ */
+export function setCenterToggle(text: string, callback: () => void) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.overlay.toggleCenter = new Toggle(text, callback);
+}
+
+/**
+ * Sets the right toggle.
+ *
+ * @param text Label on toggle.
+ * @param callback Function to call when toggle is clicked.
+ */
+export function setRightToggle(text: string, callback: () => void) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.overlay.toggleRight = new Toggle(text, callback);
+}
+
+/**
+ * Resets and hides the left toggle.
+ */
+export function removeLeftToggle() {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.overlay.toggleLeft = undefined;
+}
+
+/**
+ * Resets and hides the center toggle.
+ */
+export function removeCenterToggle() {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.overlay.toggleCenter = undefined;
+}
+
+/**
+ * Resets and hides the right toggle.
+ */
+export function removeRightToggle() {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.overlay.toggleRight = undefined;
+}
+
+// Objects
+
+/**
+ * Creates an instance of Vector3.
+ *
+ * @param x Value of x-axis.
+ * @param y Value of y-axis.
+ * @param z Value of z-axis.
+ * @returns Vector3 created from specified values.
+ */
+export function createVector3(x: number, y: number, z: number): Vector3 {
+  return new Vector3(x, y, z);
+}
+
+/**
+ * Adds the specified object to the augmented world.
+ *
+ * @param object ARObject to add. (E.g. cube, sphere, etc..)
+ */
+export function addARObject(arObject: ARObject) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  if (moduleState.arObjects.find((item) => item.id === arObject.id)) {
+    return; // Already in array
+  }
+  if (arObject.onSelect) {
+    moduleState.clickCallbacks.set(arObject.id, () => {
+      arObject.onSelect?.(arObject);
+    });
+  }
+  moduleState.arObjects.push(arObject);
+  callARCallback();
+}
+
+/**
+ * Removes the specified object from the augmented world.
+ *
+ * @param arObject ARObject to remove.
+ */
+export function removeARObject(arObject: ARObject) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.arObjects = moduleState.arObjects.filter(
+    (item) => item.id !== arObject.id,
+  );
+  callARCallback();
+}
+
+/**
+ * Obtains the current ARObjects.
+ */
+export function getARObjects(): ARObject[] {
+  const moduleState = getModuleState();
+  if (!moduleState) return [];
+  return moduleState.arObjects;
+}
+
+/**
+ * Sets AR objects from json.
+ */
+export function setAsARObjects(json: any) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  if (!(json instanceof Object)) return;
+  const objects: ARObject[] = [];
+  for (const item of Object.values(json)) {
+    const parsedObject = ARObject.fromObject(item);
+    if (parsedObject) {
+      objects.push(parsedObject);
+    }
+  }
+  moduleState.arObjects = objects;
+  callARCallback();
+}
+
+/**
+ * Removes all objects in the augmented world.
+ */
+export function clearARObjects() {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.arObjects = [];
+  callARCallback();
+}
+
+/**
+ * Obtains the position of the specified object on the x-axis.
+ *
+ * @param arObject AR object to check.
+ * @returns Value of position on the x-axis.
+ */
+export function getXPosition(arObject: ARObject): number {
+  return arObject.position.x;
+}
+/**
+ * Obtains the position of the specified object on the y-axis.
+ *
+ * @param arObject AR object to check.
+ * @returns Value of position on the y-axis.
+ */
+export function getYPosition(arObject: ARObject): number {
+  return arObject.position.y;
+}
+
+/**
+ * Obtains the position of the specified object on the z-axis.
+ *
+ * @param arObject AR object to check.
+ * @returns Value of position on the z-axis.
+ */
+export function getZPosition(arObject: ARObject): number {
+  return arObject.position.z;
+}
+
+/**
+ * Moves the specified object to a new position.
+ *
+ * @param arObject AR object to move.
+ * @param position Position to move to.
+ */
+export function moveARObject(arObject: ARObject, position: Vector3) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  arObject.position = position;
+  callARCallback();
+}
+
+// Highlight
+
+/**
+ * Turn on highlighting of object that the user is facing.
+ *
+ * @param isEnabled Whether to highlight object in front.
+ */
+export function setHighlightFrontObject(isEnabled: boolean) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.highlightFrontObject = isEnabled;
+  callARCallback();
+}
+
+/**
+ * Updates the object in line of sight.
+ *
+ * @param arObject New object to set.
+ */
+export function setFrontObject(arObject: ARObject | undefined) {
+  const moduleState = getModuleState();
+  if (!moduleState) return;
+  moduleState.selectedObjectId = arObject?.id;
+}
+
+/**
+ * Obtains the first object in the user's line of sight, if any.
+ *
+ * @returns ARObject in front of user if found, undefined if not.
+ */
+export function getFrontObject() {
+  const moduleState = getModuleState();
+  if (!moduleState) return undefined;
+  return moduleState.arObjects.find(
+    (arObject) => arObject.id === moduleState.selectedObjectId,
+  );
+}
+
+/**
+ * Sets the select state for the specified AR object.
+ *
+ * @param arObject AR object to select/unselect.
+ * @param isSelected Whether the object is selected.
+ */
+export function selectObject(arObject: ARObject, isSelected: boolean) {
+  arObject.isSelected = isSelected;
+  callARCallback();
+}
+
+// JSON
+
+/**
+ * Obtains the value of a json object at the key.
+ */
+export function getJsonChild(object: any, key: string) {
+  if (!(object instanceof Object)) return undefined;
+  return object[key];
+}