Skip to content

Commit

Permalink
fix: fine tuned lip sync and base animations
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrepat0
andrepat0 committed Oct 15, 2024
1 parent 313d23f commit 6905874
Show file tree
Hide file tree
Showing 10 changed files with 264 additions and 169 deletions.
117 changes: 81 additions & 36 deletions src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ import {
SkinnedMesh,
Object3D,
AnimationAction,
LoopOnce,
} from 'three';
import { useAnimations, useGLTF } from '@react-three/drei';
import { useGraph, dispose, useFrame } from '@react-three/fiber';
import { useGraph, useFrame } from '@react-three/fiber';
import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
import { useAvatarBlink } from '../../utils/useEyeBlink';
import { useViseme } from '../../../../../context/visemeContext';
Expand Down Expand Up @@ -43,11 +44,12 @@ const AVATAR_ROTATION = new Euler(0.175, 0, 0);
const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);

const ANIMATION_URLS = {
MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb',
MALE: 'https://assets.memori.ai/api/v2/asset/2c5e88a4-cf62-408b-9ef0-518b099dfcb2.glb',
FEMALE:
'https://assets.memori.ai/api/v2/asset/c2b07166-de10-4c66-918b-7b7cd380cca7.glb',
'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
};
const ANIMATION_DURATION = 3000; // Duration in milliseconds for non-idle animations

const TRANSITION_DURATION = 0.5; // Duration for transitioning between animations

export default function FullbodyAvatar({
url,
Expand All @@ -61,7 +63,7 @@ export default function FullbodyAvatar({
morphTargetInfluences,
eyeBlink,
setMeshRef,
clearVisemes,
//clearVisemes,
}: FullbodyAvatarProps) {
const { scene } = useGLTF(url);
const { animations } = useGLTF(ANIMATION_URLS[sex]);
Expand All @@ -72,7 +74,8 @@ export default function FullbodyAvatar({
const avatarMeshRef = useRef<SkinnedMesh>();
const currentActionRef = useRef<AnimationAction | null>(null);
const isTransitioningRef = useRef(false);

const lastActionTimeRef = useRef(0);
const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
// Blink animation
useAvatarBlink({
enabled: eyeBlink || false,
Expand All @@ -92,7 +95,10 @@ export default function FullbodyAvatar({

const finishCurrentAnimation = () => {
if (currentActionRef.current && !currentActionRef.current.paused) {
const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000;
const remainingTime =
(currentActionRef.current.getClip().duration -
currentActionRef.current.time) *
1000;
setTimeout(() => {
startIdleAnimation();
}, remainingTime);
Expand All @@ -109,22 +115,28 @@ export default function FullbodyAvatar({
idleAnimations[Math.floor(Math.random() * idleAnimations.length)];

const idleAction = actions[randomIdle];
const fadeOutDuration = 0.5;
const fadeInDuration = 0.5;

if (currentActionRef.current) {
currentActionRef.current.fadeOut(fadeOutDuration);
if (currentActionRef.current && idleAction) {
currentActionRef.current.crossFadeTo(
idleAction,
TRANSITION_DURATION,
true
);
}

idleAction?.reset().fadeIn(fadeInDuration).play();
currentActionRef.current = idleAction;
if (idleAction) {
idleAction.reset().fadeIn(TRANSITION_DURATION).play();
currentActionRef.current = idleAction;
}

setTimeout(() => {
isTransitioningRef.current = false;
}, (fadeOutDuration + fadeInDuration) * 1000);
}, TRANSITION_DURATION * 1000);
};

if (currentActionRef.current && !currentActionRef.current.getClip().name.startsWith('Idle')) {
if (
currentActionRef.current &&
!currentActionRef.current.getClip().name.startsWith('Idle')
) {
finishCurrentAnimation();
} else {
startIdleAnimation();
Expand All @@ -133,8 +145,7 @@ export default function FullbodyAvatar({

// Base animation
useEffect(() => {
if (!actions || !currentBaseAction.action || isTransitioningRef.current)
return;
if (!actions || !currentBaseAction.action) return;

const newAction = actions[currentBaseAction.action];
if (!newAction) {
Expand All @@ -147,20 +158,30 @@ export default function FullbodyAvatar({
const fadeOutDuration = 0.8;
const fadeInDuration = 0.8;

if (!currentBaseAction.action.startsWith('Idle')) {
setTimeout(() => {
transitionToIdle();
}, ANIMATION_DURATION);
}

if (currentActionRef.current) {
currentActionRef.current.fadeOut(fadeOutDuration);
}

newAction.timeScale = timeScale;
console.log(newAction);
newAction.reset().fadeIn(fadeInDuration).play();
currentActionRef.current = newAction;
}, [currentBaseAction, timeScale, actions, transitionToIdle]);

// Set the time scale for the new action
newAction.timeScale = timeScale;

// If it's an emotion animation, set it to play once and then transition to idle
if (
currentBaseAction.action.startsWith('Gioia') ||
currentBaseAction.action.startsWith('Rabbia') ||
currentBaseAction.action.startsWith('Sorpresa') ||
currentBaseAction.action.startsWith('Timore') ||
currentBaseAction.action.startsWith('Tristezza')
) {
newAction.setLoop(LoopOnce, 1);
newAction.clampWhenFinished = true;
setIsTransitioningToIdle(true);
}
}, [actions, currentBaseAction, timeScale]);

// Set up the mesh reference and morph target influences
useEffect(() => {
Expand Down Expand Up @@ -188,9 +209,11 @@ export default function FullbodyAvatar({
onLoaded?.();

return () => {
Object.values(materials).forEach(dispose);
Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
clearVisemes();
Object.values(materials).forEach(material => material.dispose());
Object.values(nodes)
.filter(isSkinnedMesh)
.forEach(mesh => mesh.geometry.dispose());
mixer.stopAllAction();
};
}, [
materials,
Expand All @@ -200,27 +223,49 @@ export default function FullbodyAvatar({
setMorphTargetDictionary,
setMorphTargetInfluences,
setMeshRef,
clearVisemes,
mixer,
]);

// Update morph target influences
useFrame((_, delta) => {
if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
updateMorphTargetInfluences();
}
mixer.update(delta * 0.001);
mixer.update(delta);

function updateMorphTargetInfluences() {
Object.entries(morphTargetInfluences).forEach(([key, value]) => {
const index = avatarMeshRef.current!.morphTargetDictionary![key];
if (typeof index === 'number' &&
avatarMeshRef.current!.morphTargetInfluences) {
const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
if (
typeof index === 'number' &&
avatarMeshRef.current!.morphTargetInfluences
) {
const currentValue =
avatarMeshRef.current!.morphTargetInfluences[index];
const smoothValue = lerp(currentValue, value, 0.1);
avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
}
});
}

// Handle transition from emotion animation to idle
if (isTransitioningToIdle && currentActionRef.current) {
if (
currentActionRef.current.time >=
currentActionRef.current.getClip().duration
) {
// Transition to the idle animation
const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];

if (idleAction) {
currentActionRef.current.fadeOut(0.5);
idleAction.reset().fadeIn(0.5).play();
currentActionRef.current = idleAction;
setIsTransitioningToIdle(false);
}
}
}
});

return (
Expand All @@ -231,4 +276,4 @@ export default function FullbodyAvatar({
<primitive object={scene} />
</group>
);
}
}
5 changes: 3 additions & 2 deletions src/components/MemoriWidget/MemoriWidget.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1981,12 +1981,12 @@ const MemoriWidget = ({
userLang
)}"><voice name="${getTTSVoice(
userLang
)}"><mstts:express-as style="${getAzureStyleForEmotion(
)}"><prosody rate="0.95"><mstts:express-as style="${getAzureStyleForEmotion(
emotion
)}"><s>${replaceTextWithPhonemes(
textToSpeak,
userLang.toLowerCase()
)}</s></mstts:express-as></voice></speak>`,
)}</s></mstts:express-as></prosody></voice></speak>`,
result => {
if (result) {
setIsPlayingAudio(true);
Expand Down Expand Up @@ -2014,6 +2014,7 @@ const MemoriWidget = ({
) {
source.disconnect();
setIsPlayingAudio(false);
clearVisemes();
memoriSpeaking = false;
} else if ((audioContext.state as string) === 'interrupted') {
audioContext.resume();
Expand Down
2 changes: 1 addition & 1 deletion src/components/layouts/ZoomedFullBody.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@ it('renders ZOOMED_FULL_BODY layout unchanged', () => {
</I18nWrapper>
);
expect(container).toMatchSnapshot();
});
});
67 changes: 38 additions & 29 deletions src/components/layouts/ZoomedFullBody.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import React from 'react';
import React, { useEffect, useCallback } from 'react';
import Spin from '../ui/Spin';
import { LayoutProps } from '../MemoriWidget/MemoriWidget';

const FullPageLayout: React.FC<LayoutProps> = ({
const ZoomedFullBodyLayout: React.FC<LayoutProps> = ({
Header,
headerProps,
Avatar,
Expand All @@ -20,36 +20,45 @@ const FullPageLayout: React.FC<LayoutProps> = ({
showInstruct = false,
loading = false,
poweredBy,
}) => (
<>
{integrationStyle}
{integrationBackground}
}) => {
useEffect(() => {
document.body.style.overflow = 'hidden';
return () => {
document.body.style.overflow = '';
};
}, []);

<Spin spinning={loading}>
{showInstruct && ChangeMode && changeModeProps && (
<ChangeMode {...changeModeProps} />
)}
return (
<>
{integrationStyle}
{integrationBackground}

{Header && headerProps && <Header {...headerProps} />}
<Spin className="memori-spin--zoomed-full-body" spinning={loading}>
{showInstruct && ChangeMode && changeModeProps && <ChangeMode {...changeModeProps} />}

<div className="memori--grid">
<div className="memori--grid-column memori--grid-column-left">
{Avatar && avatarProps && <Avatar chatProps={chatProps} isZoomed {...avatarProps} />}
{Header && headerProps && <Header {...headerProps} />}

<div id="extension" />
</div>
<div className="memori--grid-column memori--grid-column-right">
{sessionId && hasUserActivatedSpeak && Chat && chatProps ? (
<Chat {...chatProps} />
) : startPanelProps ? (
<StartPanel {...startPanelProps} />
) : null}
</div>
<div className="memori--grid">
<div className="memori--grid-column memori--grid-column-left">
{Avatar && avatarProps && (
<Avatar chatProps={chatProps} isZoomed {...avatarProps} />
)}

{poweredBy}
</div>
</Spin>
</>
);
<div id="extension" />
</div>
<div className="memori--grid-column memori--grid-column--zoomed-full-body memori--grid-column-right">
{sessionId && hasUserActivatedSpeak && Chat && chatProps ? (
<Chat {...chatProps} />
) : startPanelProps ? (
<StartPanel {...startPanelProps} />
) : null}
</div>

<div className="memori--powered-by-custom">{poweredBy}</div>
</div>
</Spin>
</>
);
};

export default FullPageLayout;
export default ZoomedFullBodyLayout;
Loading

0 comments on commit 6905874

Please sign in to comment.