From dd8686bd0bcf6df5f09a4b0458c135f8f3f57335 Mon Sep 17 00:00:00 2001 From: Kathy Zhang <112294375+b27b@users.noreply.github.com> Date: Fri, 11 Oct 2024 12:54:05 -0400 Subject: [PATCH] update anima --- app/(pages)/projects/(projects)/anima/page.js | 89 ++++++------------- 1 file changed, 29 insertions(+), 60 deletions(-) diff --git a/app/(pages)/projects/(projects)/anima/page.js b/app/(pages)/projects/(projects)/anima/page.js index a71be0e..110a633 100644 --- a/app/(pages)/projects/(projects)/anima/page.js +++ b/app/(pages)/projects/(projects)/anima/page.js @@ -5,82 +5,51 @@ import ProjDescrip from "../../ProjDescrip"; import JoinUs from "@/app/components/JoinUs"; import BackButton from "@/app/components/BackButton"; -import Anima from "@/public/images/photos/anima/anima.png"; +import HeaderImage from "@/public/images/photos/anima/anima.png"; -export default function UniversalGestures() { +export default function Anima() { return (
- +
- When you prompt a virtual assistant (for example Meta AI on - Raybans glasses), what happens when you ask “What am I looking - at”? Currently, the pipeline seems rather simplistic. The cameras - on the glasses take a picture, that picture is passed through a - model that can assign text labels to images, and finally that text - label describing the whole image is passed into an LLM. This - process, especially the step where a model must describe - everything in an image using words, is often inaccurate. +

+ We are creating a Unity VR application that allows + users to explore real-life building scans in true scale using a + Meta Quest headset. The application will focus on photogrammetry + scans sourced primarily from OpenHeritage3D, providing detailed + and accurate models of culturally significant buildings.

, -

- What if we could build a system that… -
- …provides a richer text summary of a virtual environment, complete - with descriptions of how objects compose each other, are placed - within/next to/on top of each other? -
- …also describes how you, the user, is interacting with that - environment at any moment? Could we assign additional text to - describe that you are pointing at a specific object, or reaching - out for one? -
- …runs in real time, that is, can constantly update every frame to - provide an updated description. That way, we wouldn't have to - wait for text generation, and we could create a live captioning - system? …runs entirely on-device, meaning this information is - never sent to the cloud? +

+ The project will involve importing and optimising these 3D scans, + integrating VR interaction systems, and ensuring smooth + performance on Meta Quest devices. The end goal is to deliver an + immersive and realistic virtual experience where users can walk + through and interact with real-world buildings as if they were + physically present.

, -

- If we created this, we could use it for… -
- …in-application virtual assistants that make use of a rich text - summary for high-accuracy responses -
- …virtual science labs where users could receive detailed - auto-generated scientific explanations about tools and objects - they interact with -
- …dynamic VR scene descriptions for the visually impaired, - describing layout and objects, or even what they're holding, - pointing at or nearby to -
- …and so much more -

, -

- Universal Text aims to explore this. We are creating a structured - software package for Unity that allows for real time captioning of - a VR user's interactions with their virtual environment. In - other words, tools provided by our package aim to describe in - natural language "what's happening" in a VR application at - any moment in time, as if recounted by a third party observer. - This textual description will be rich in detail and generated - on-the-fly, providing seamless integration of tutorials, live - captioning for accessibility, or virtual assistants into VR - applications. +

+ From the users perspective, they would be able to download the + Anima application either on the Meta Quest storefront or through + other means, select one of the preloaded scans or import one into + the application, and then enter a rendering of the building and be + able to view it from multiple different angles through the VR + headset. This would include a 360 degree view of the rendered + building and the application responding to the orientation of the + headset.

, ]} - link="https://github.com/uwrealitylabs/universal-gestures-unity" + // link="https://github.com/uwrealitylabs/universal-gestures-unity" />