feat: improve lip sync (#20)

* feat: setted the avatar to cover all the screen for the ZOOMED FULL BODY layout * feat: improved stripOutputTags function * refactor: simplified viseme and animation process for lip sync * feat: fine tuned lips values for more smoothness * refactor: improve speak function * refactor: inserted blink and emotion animation all in the same useFrame * refactor: resetted previous emotion morph targets when new emotion is triggered * fix: adjusted position of input and powered by box inside the ZOOMED FULL BODY layout
memori-ai · Oct 15, 2024 · e13f520 · e13f520
1 parent 78f1fa4
commit e13f520
Show file tree

Hide file tree

Showing 12 changed files with 578 additions and 657 deletions.
diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx
@@ -57,7 +57,7 @@ const Avatar: React.FC<Props> = ({
   const { t } = useTranslation();
   const [isClient, setIsClient] = useState(false);
 
-  const { setMeshRef, clearVisemes, setEmotion } = useViseme();
+  const { stopProcessing, updateCurrentViseme, resetVisemeQueue } = useViseme();
 
   useEffect(() => {
     setIsClient(true);
@@ -132,6 +132,7 @@ const Avatar: React.FC<Props> = ({
           }
         >
           <ContainerAvatarView
+            updateCurrentViseme={updateCurrentViseme}
             url={integrationConfig.avatarURL}
             sex={memori.voiceType === 'FEMALE' ? 'FEMALE' : 'MALE'}
             fallbackImg={getAvatarUrl()}
@@ -142,11 +143,10 @@ const Avatar: React.FC<Props> = ({
             speaking={isPlayingAudio}
             loading={loading}
             style={getAvatarStyle()}
-            clearVisemes={clearVisemes}
-            setMeshRef={setMeshRef}
-            isZoomed={isZoomed}
+            stopProcessing={stopProcessing}
+            resetVisemeQueue={resetVisemeQueue}
+            isZoomed={isZoomed} 
             chatEmission={chatProps?.dialogState?.emission}
-            setEmotion={setEmotion}
           />
         </ErrorBoundary>
       );

diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
@@ -14,9 +14,9 @@ interface Props {
   speaking: boolean;
   isZoomed: boolean;
   chatEmission: any;
-  setMeshRef: any;
-  clearVisemes: () => void;
-  setEmotion: (emotion: string) => void;
+  stopProcessing: () => void;
+  resetVisemeQueue: () => void;
+  updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
 }
 
 interface BaseAction {
@@ -50,9 +50,9 @@ const baseActions: Record<string, BaseAction> = {
   Loading3: { weight: 0 },
 };
 
+
 export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
-  setMeshRef,
-  clearVisemes,
+  stopProcessing,
   chatEmission,
   showControls,
   animation,
@@ -64,7 +64,8 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   halfBody,
   loading,
   isZoomed,
-  setEmotion,
+  updateCurrentViseme,
+  resetVisemeQueue,
 }) => {
   const [currentBaseAction, setCurrentBaseAction] = useState({
     action: animation || 'Idle1',
@@ -77,6 +78,9 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   const [morphTargetDictionary, setMorphTargetDictionary] = useState<{
     [key: string]: number;
   }>({});
+  const [emotionMorphTargets, setEmotionMorphTargets] = useState<{
+    [key: string]: number;
+  }>({});
 
   const [timeScale, setTimeScale] = useState(0.8);
 
@@ -93,7 +97,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
 
     //remove the last character from the action
     const newEmotion = action.slice(0, -1);
-    setEmotion(newEmotion);
+    // setEmotion(newEmotion);
 
     const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => {
       acc[key] = 0;
@@ -105,9 +109,8 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
     const emotionValues =
       emotion === 'default' ? defaultEmotions : emotionMap[emotion];
 
-    setMorphTargetInfluences(prevInfluences => ({
-      ...prevInfluences,
-      ...defaultEmotions,
+    setEmotionMorphTargets(prevEmotions => ({
+      ...prevEmotions,
       ...emotionValues,
     }));
   }, []);
@@ -162,6 +165,11 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       const emotion = `${outputContent}${randomNumber}`;
 
       onBaseActionChange(emotion);
+    } else {
+      //Set a random idle animation
+      const randomNumber = Math.floor(Math.random() * 5) + 1;
+      const animation = `Idle${randomNumber === 3 ? 4 : randomNumber}`;
+      onBaseActionChange(animation);
     }
   }, [chatEmission]);
 
@@ -174,6 +182,13 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
     }
   }, [loading]);
 
+  // useEffect(() => {
+  //   if (speaking && currentBaseAction.action !== 'Idle1') {
+  //     const animation = `Idle1`;
+  //     onBaseActionChange(animation);
+  //   }
+  // }, [speaking]);
+
   return (
     <>
       {showControls && (
@@ -191,30 +206,28 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       {halfBody ? (
         <HalfBodyAvatar
           url={url}
-          setMeshRef={setMeshRef}
-          setMorphTargetInfluences={setMorphTargetInfluences}
           headMovement={headMovement}
           speaking={speaking}
           eyeBlink={eyeBlink}
           morphTargetInfluences={morphTargetInfluences}
-          clearVisemes={clearVisemes}
+          setMorphTargetInfluences={setMorphTargetInfluences}
           setMorphTargetDictionary={setMorphTargetDictionary}
         />
       ) : (
         <FullbodyAvatar
           url={url}
           sex={sex}
+          resetVisemeQueue={resetVisemeQueue}
           eyeBlink={eyeBlink}
-          speaking={speaking}
           currentBaseAction={currentBaseAction}
           timeScale={timeScale}
-          setMorphTargetInfluences={setMorphTargetInfluences}
-          setMorphTargetDictionary={setMorphTargetDictionary}
           morphTargetInfluences={morphTargetInfluences}
-          morphTargetDictionary={morphTargetDictionary}
           isZoomed={isZoomed}
-          setMeshRef={setMeshRef}
-          clearVisemes={clearVisemes}
+          updateCurrentViseme={updateCurrentViseme}
+          stopProcessing={stopProcessing}
+          setMorphTargetDictionary={setMorphTargetDictionary}
+          setMorphTargetInfluences={setMorphTargetInfluences}
+          emotionMorphTargets={emotionMorphTargets}
         />
       )}
     </>