Skip to content

Commit

Permalink
feat: improved fullbody avatar state management
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrepat0
andrepat0 committed Oct 17, 2024
1 parent fb37107 commit 3360706
Showing 1 changed file with 86 additions and 92 deletions.
178 changes: 86 additions & 92 deletions src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import React, { useEffect, useRef, useState } from 'react';
import React, { useEffect, useRef, useMemo, useCallback } from 'react';
import {
Vector3,
Euler,
Expand All @@ -11,7 +11,6 @@ import {
} from 'three';
import { useAnimations, useGLTF } from '@react-three/drei';
import { useGraph, useFrame } from '@react-three/fiber';
import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';

interface FullbodyAvatarProps {
url: string;
Expand Down Expand Up @@ -41,6 +40,7 @@ interface FullbodyAvatarProps {
emotionMorphTargets: Record<string, number>;
}


const AVATAR_POSITION = new Vector3(0, -1, 0);
const AVATAR_ROTATION = new Euler(0.175, 0, 0);
const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
Expand All @@ -51,80 +51,75 @@ const ANIMATION_URLS = {
'https://assets.memori.ai/api/v2/asset/0e49aa5d-f757-4292-a170-d843c2839a41.glb',
};

// Blink configuration
const BLINK_CONFIG = {
minInterval: 1000,
maxInterval: 5000,
blinkDuration: 150,
};

const EMOTION_TRANSITION_SPEED = 0.1; // Adjust this value to control emotion transition speed
const EMOTION_SMOOTHING = 0.3;
const VISME_SMOOTHING = 0.5;


export default function FullbodyAvatar({
url,
sex,
onLoaded,
currentBaseAction,
timeScale,
isZoomed,
eyeBlink,
morphTargetSmoothing = 0.5,
updateCurrentViseme,
setMorphTargetDictionary,
setMorphTargetInfluences,
emotionMorphTargets,
}: FullbodyAvatarProps) {
const { scene } = useGLTF(url);
const { animations } = useGLTF(ANIMATION_URLS[sex]);
const { nodes, materials } = useGraph(scene);
const { actions } = useAnimations(animations, scene);

const mixer = useRef(new AnimationMixer(scene));
const mixerRef = useRef<AnimationMixer>();
const headMeshRef = useRef<SkinnedMesh>();
const currentActionRef = useRef<AnimationAction | null>(null);
const [isTransitioningToIdle, setIsTransitioningToIdle] = useState(false);
const isTransitioningToIdleRef = useRef(false);

// Blink state
const lastBlinkTime = useRef(0);
const nextBlinkTime = useRef(0);
const isBlinking = useRef(false);
const blinkStartTime = useRef(0);
const lastBlinkTimeRef = useRef(0);
const nextBlinkTimeRef = useRef(0);
const isBlinkingRef = useRef(false);
const blinkStartTimeRef = useRef(0);

// Morph targets
const currentEmotionRef = useRef<Record<string, number>>({});
const previousEmotionKeysRef = useRef<Set<string>>(new Set());

useEffect(() => {
correctMaterials(materials);

// Memoize the scene traversal
const headMesh = useMemo(() => {
let foundMesh: SkinnedMesh | undefined;
scene.traverse((object: Object3D) => {
if (object instanceof SkinnedMesh) {
if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') {
headMeshRef.current = object;
if (object.morphTargetDictionary && object.morphTargetInfluences) {
setMorphTargetDictionary(object.morphTargetDictionary);

const initialInfluences = Object.keys(
object.morphTargetDictionary
).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
setMorphTargetInfluences(initialInfluences);
}
}
if (
object instanceof SkinnedMesh &&
(object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
) {
foundMesh = object;
}
});
return foundMesh;
}, [scene]);

onLoaded?.();

return () => {
Object.values(materials).forEach(material => material.dispose());
Object.values(nodes)
.filter(isSkinnedMesh)
.forEach(mesh => mesh.geometry.dispose());
};
}, [materials, nodes, url, onLoaded, scene]);

// Handle base animation changes
useEffect(() => {
if (headMesh) {
headMeshRef.current = headMesh;
if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) {
setMorphTargetDictionary(headMesh.morphTargetDictionary);
const initialInfluences = Object.keys(
headMesh.morphTargetDictionary
).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
setMorphTargetInfluences(initialInfluences);
}
}
mixerRef.current = new AnimationMixer(scene);
}, [headMesh, scene, setMorphTargetDictionary, setMorphTargetInfluences]);

// Memoize the animation change handler
const handleAnimationChange = useCallback(() => {
if (!actions || !currentBaseAction.action) return;

const newAction = actions[currentBaseAction.action];
Expand All @@ -141,15 +136,11 @@ export default function FullbodyAvatar({
if (currentActionRef.current) {
currentActionRef.current.fadeOut(fadeOutDuration);
}

console.log(newAction);

newAction.reset().fadeIn(fadeInDuration).play();
currentActionRef.current = newAction;

// Set the time scale for the new action
newAction.timeScale = timeScale;

// If it's an emotion animation, set it to play once and then transition to idle
if (
currentBaseAction.action.startsWith('Gioia') ||
currentBaseAction.action.startsWith('Rabbia') ||
Expand All @@ -159,55 +150,55 @@ export default function FullbodyAvatar({
) {
newAction.setLoop(LoopOnce, 1);
newAction.clampWhenFinished = true;
setIsTransitioningToIdle(true);
isTransitioningToIdleRef.current = true;
}
}, [actions, currentBaseAction, timeScale]);

useFrame(state => {
if (
headMeshRef.current &&
headMeshRef.current.morphTargetDictionary &&
headMeshRef.current.morphTargetInfluences
) {
const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds
useEffect(() => {
handleAnimationChange();
}, [handleAnimationChange]);

// Optimize the frame update function
const updateFrame = useCallback(
(currentTime: number) => {
if (
!headMeshRef.current ||
!headMeshRef.current.morphTargetDictionary ||
!headMeshRef.current.morphTargetInfluences
)
return;

// Handle blinking
let blinkValue = 0;
if (eyeBlink) {
if (currentTime >= nextBlinkTime.current && !isBlinking.current) {
isBlinking.current = true;
blinkStartTime.current = currentTime;
lastBlinkTime.current = currentTime;
nextBlinkTime.current =
if (currentTime >= nextBlinkTimeRef.current && !isBlinkingRef.current) {
isBlinkingRef.current = true;
blinkStartTimeRef.current = currentTime;
lastBlinkTimeRef.current = currentTime;
nextBlinkTimeRef.current =
currentTime +
Math.random() *
(BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) +
BLINK_CONFIG.minInterval;
}

if (isBlinking.current) {
if (isBlinkingRef.current) {
const blinkProgress =
(currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration;
(currentTime - blinkStartTimeRef.current) /
BLINK_CONFIG.blinkDuration;
if (blinkProgress <= 0.5) {
// Eyes closing
blinkValue = blinkProgress * 2;
} else if (blinkProgress <= 1) {
// Eyes opening
blinkValue = 2 - blinkProgress * 2;
} else {
// Blink finished
isBlinking.current = false;
isBlinkingRef.current = false;
blinkValue = 0;
}
}
}

const currentViseme = updateCurrentViseme(currentTime / 1000);

// Create a set of current emotion keys
const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets));

// Reset old emotion morph targets
previousEmotionKeysRef.current.forEach(key => {
if (!currentEmotionKeys.has(key)) {
const index = headMeshRef.current!.morphTargetDictionary![key];
Expand All @@ -220,75 +211,78 @@ export default function FullbodyAvatar({
}
});

// Update morph targets
Object.entries(headMeshRef.current.morphTargetDictionary).forEach(
([key, index]) => {
if (typeof index === 'number') {
let targetValue = 0;

// Handle emotions (base layer)
if (Object.prototype.hasOwnProperty.call(emotionMorphTargets, key)) {
if (currentEmotionKeys.has(key)) {
const targetEmotionValue = emotionMorphTargets[key];
const currentEmotionValue = currentEmotionRef.current[key] || 0;
const newEmotionValue = MathUtils.lerp(
currentEmotionValue,
targetEmotionValue * 2,
EMOTION_TRANSITION_SPEED
targetEmotionValue * 2.5,
EMOTION_SMOOTHING
);
currentEmotionRef.current[key] = newEmotionValue;
targetValue += newEmotionValue;
}

// Handle visemes (additive layer)
if (currentViseme && key === currentViseme.name) {
targetValue += currentViseme.weight * 1.3; // Amplify the effect
targetValue += currentViseme.weight * 1;
}

// Handle blinking (additive layer, only for 'eyesClosed')
if (key === 'eyesClosed' && eyeBlink) {
targetValue += blinkValue;
}

// Clamp the final value between 0 and 1
targetValue = MathUtils.clamp(targetValue, 0, 1);

// Apply smoothing
if (headMeshRef.current && headMeshRef.current.morphTargetInfluences) {
if (
headMeshRef.current &&
headMeshRef.current.morphTargetInfluences
) {
headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp(
headMeshRef.current.morphTargetInfluences[index],
targetValue,
morphTargetSmoothing
VISME_SMOOTHING
);
}
}
}
);

// Update the set of previous emotion keys for the next frame
previousEmotionKeysRef.current = currentEmotionKeys;

// Handle transition from emotion animation to idle
if (isTransitioningToIdle && currentActionRef.current) {
if (isTransitioningToIdleRef.current && currentActionRef.current) {
if (
currentActionRef.current.time >=
currentActionRef.current.getClip().duration
) {
// Transition to the idle animation
const idleNumber = Math.floor(Math.random() * 5) + 1; // Randomly choose 1, 2, 3, 4 or 5
const idleAction = actions[`Idle${idleNumber == 3 ? 4 : idleNumber}`];
const idleNumber = Math.floor(Math.random() * 5) + 1;
const idleAction =
actions[`Idle${idleNumber === 3 ? 4 : idleNumber}`];

if (idleAction) {
currentActionRef.current.fadeOut(0.5);
idleAction.reset().fadeIn(0.5).play();
currentActionRef.current = idleAction;
setIsTransitioningToIdle(false);
isTransitioningToIdleRef.current = false;
}
}
}

// Update the animation mixer
mixer.current.update(0.01); // Fixed delta time for consistent animation speed
}
mixerRef.current?.update(0.01);
},
[
actions,
emotionMorphTargets,
eyeBlink,
updateCurrentViseme,
]
);

useFrame(state => {
updateFrame(state.clock.getElapsedTime() * 1000);
});

return (
Expand Down

0 comments on commit 3360706

Please sign in to comment.