feat: fine tuned lip sync

memori-ai · Oct 16, 2024 · f8557ee · f8557ee
1 parent 3626b55
commit f8557ee
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 7 deletions.
diff --git a/src/components/MemoriWidget/MemoriWidget.tsx b/src/components/MemoriWidget/MemoriWidget.tsx
@@ -1963,6 +1963,8 @@ const MemoriWidget = ({
       onEndSpeakStartListen();
     };
 
+    // Add this before starting new speech synthesis
+    resetVisemeQueue();
 
     // Set up the viseme event handler
     speechSynthesizer.visemeReceived = function (_, e) {
@@ -1976,9 +1978,7 @@ const MemoriWidget = ({
     speechSynthesizer.speakSsmlAsync(
       `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
         userLang
-      )}"><voice name="${getTTSVoice(
-        userLang
-      )}"><s>${replaceTextWithPhonemes(
+      )}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(
         textToSpeak,
         userLang.toLowerCase()
       )}</s></voice></speak>`,
@@ -1990,13 +1990,15 @@ const MemoriWidget = ({
           try {
             // Decode the audio data
             audioContext!.decodeAudioData(result.audioData, function (buffer) {
+              console.log('decoded');
               const currentSource = audioContext!.createBufferSource();
               currentSource.buffer = buffer;
               currentSource.connect(audioContext!.destination);
               startProcessing();
               currentSource.start();
 
               currentSource.onended = () => {
+                console.log('ended');
                 setIsPlayingAudio(false);
                 memoriSpeaking = false;
                 stopProcessing();
@@ -2011,20 +2013,22 @@ const MemoriWidget = ({
                 audioContext.state === 'suspended' ||
                 audioContext.state === 'closed'
               ) {
+                console.log('suspended');
                 source.disconnect();
                 setIsPlayingAudio(false);
                 stopProcessing();
                 resetVisemeQueue();
                 memoriSpeaking = false;
                 emitEndSpeakEvent();
               } else if ((audioContext.state as string) === 'interrupted') {
+                console.log('interrupted');
                 stopProcessing();
                 resetVisemeQueue();
+
                 audioContext.resume();
               }
             };
 
-
           } catch (e) {
             console.warn('speak error: ', e);
             window.speechSynthesis.speak(new SpeechSynthesisUtterance(text));
@@ -3420,4 +3424,4 @@ const MemoriWidget = ({
   );
 };
 
-export default MemoriWidget;
+export default MemoriWidget;
diff --git a/src/context/visemeContext.tsx b/src/context/visemeContext.tsx
@@ -49,7 +49,7 @@ const DEFAULT_VISEME_DURATION = 0.04 //0; // Reduced from 0.4 for smoother trans
 const VISEME_OVERLAP = 0.35; // Slightly increased from 0.04 for more overlap
 const SMOOTHING_FACTOR = 0.35 // New constant for weight smoothing
 const TIME_OFFSET = -0.04; // Adjust this value as needed (in seconds)
-const PRELOAD_TIME = 0.5; // Preload visemes 0.5 seconds in advance
+const PRELOAD_TIME = 1; // Preload visemes 0.5 seconds in advance
 
 export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
   const visemeQueueRef = useRef<Viseme[]>([]);
@@ -118,7 +118,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
   }, [isProcessing]);
 
   const startProcessing = useCallback(() => {
-    if (isProcessing) return;
+    // if (isProcessing) return;
     startTimeRef.current = performance.now() / 1000;
     setIsProcessing(true);
   }, [isProcessing]);