Merge pull request #33 from chiefeu/add-audio

added audio play
jtoy · Mar 27, 2024 · d603072 · d603072
2 parents 75580b1 + 568b9e0
commit d603072
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 5 deletions.
diff --git a/web/src/components/RenderLevel.tsx b/web/src/components/RenderLevel.tsx
@@ -36,6 +36,7 @@ const RenderLevel: React.FC<{ simId: string, map?: string | null, img?: string |
     const [initialFetchDone, setInitialFetchDone] = useState(false);
     const chunkSize = 1000; // Adjust chunk size as needed
 
+
     const levelRef = useRef<Level>(new Level([], (newState: LevelState) => {
         setLevelState(newState);
     }));
@@ -177,7 +178,8 @@ const RenderLevel: React.FC<{ simId: string, map?: string | null, img?: string |
                     setIsPlaying={setIsPlaying}
                     stepId={levelState.stepId}
                     substepId={levelState.substepId}
-                    level={levelRef.current} />
+                    level={levelRef.current}
+                    simId={simId} />
             </div>
         </div>
     );

diff --git a/web/src/components/Sidebar.module.css b/web/src/components/Sidebar.module.css
@@ -52,4 +52,10 @@
     display: flex; 
     align-items: center; 
     justify-content: flex-end;
+}
+
+.stepAndAudio {
+    display: flex;
+    flex-direction: row;
+    gap: 12px;
 }
diff --git a/web/src/components/Sidebar.tsx b/web/src/components/Sidebar.tsx
@@ -16,6 +16,7 @@ interface SidebarProps {
     stepId: number;
     substepId: number;
     level: Level;
+    simId: string;
 }
 
 const Sidebar: React.FC<SidebarProps> = (
@@ -26,11 +27,18 @@ const Sidebar: React.FC<SidebarProps> = (
         setIsPlaying,
         stepId, 
         substepId, 
-        level 
+        level,
+        simId
     }) => {
 
-    const [showThoughts, setShowThoughts] = useState(true);
-
+        const [showThoughts, setShowThoughts] = useState(true);
+        const [isPlayAudio, setIsPlayAudio] = useState(true);
+        const [audioQueue, setAudioQueue] = useState<Promise<string>[]>([]);
+        const [audioPlaying, setAudioPlaying] = useState<boolean>(false);
+        const browserLanguage = navigator.language;
+
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const minimal_audio_delay = 500; // delay in between playing audio clips
 
     useEffect(() => {
         let interval: NodeJS.Timeout;
@@ -48,6 +56,81 @@ const Sidebar: React.FC<SidebarProps> = (
         };
     }, [isPlaying]);
 
+    const fetchAudioData = async (sim_id: string, step_id: number, substep_id: number, agent_name: string, lang: string): Promise<string> => {
+        try {
+            const res = await fetch(
+                `${process.env.NEXT_PUBLIC_ASSET_DOMAIN}/audio?mid=${sim_id}&step=${step_id}&substep=${substep_id}&agent=${agent_name}&lang=${lang}`,
+                { mode: 'cors' }
+            );
+            if (!res.ok) {
+                throw new Error('Failed to fetch data');
+            }
+
+            const audioBlob = await res.blob();
+            const audioUrl = URL.createObjectURL(audioBlob);
+            return audioUrl;
+        } catch (error) {
+            console.error('Error fetching audio:', error);
+            return "";
+        }
+    };
+
+    const addToAudioQueue = (audioClipUrl: Promise<string>) => {
+        setAudioQueue((oldQueue) => [...oldQueue, audioClipUrl]);
+    };
+
+    const playAudio = async (audioClipUrl: Promise<string>) => {
+        console.log("BEFORE LENGTH", audioQueue.length)
+        setAudioPlaying(true);
+        const audio = new Audio(await audioClipUrl);
+        audio.onended = () => {
+            setAudioQueue((oldQueue) => oldQueue.slice(1));
+            console.log("AFTER LENGTH", audioQueue.length)
+            setAudioPlaying(false);
+        };
+        audio.play();
+    };
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const stopAudio = (audio: HTMLAudioElement) => {
+        if (audio) {
+            audio.pause();
+            audio.currentTime = 0;
+        }
+    };
+
+    useEffect(() => {
+        if (isPlaying && agentPlacement && !audioPlaying && audioQueue.length > 0) {
+            playAudio(audioQueue[0]);
+        }
+    }, [agentPlacement, isPlaying, audioQueue, audioPlaying]);
+
+    useEffect(() => {
+        if (agentPlacement && isPlayAudio) {
+            const steps = agentPlacement.steps.filter(
+                step => step.stepId >= stepId
+            );
+            steps.forEach(step => {
+                if (step instanceof TalkStep) {
+                    const talkStep = step as TalkStep;
+                    addToAudioQueue(fetchAudioData(simId, talkStep.stepId, talkStep.substepId, talkStep.fromAgentName, browserLanguage));
+
+                    return;
+                }
+                if (showThoughts && step instanceof ThoughtStep) {
+                    const thoughtStep = step as ThoughtStep;
+                    addToAudioQueue(fetchAudioData(simId, thoughtStep.stepId, thoughtStep.substepId, thoughtStep.agentId, browserLanguage));
+                    return;
+                }
+            });
+        }
+
+        if (!agentPlacement || !isPlayAudio) {
+            setAudioQueue([]);
+        }
+
+    }, [agentPlacement, showThoughts, isPlayAudio, stepId, substepId]);
+
     const handleRewind = (): void => {
         setIsPlaying(false);
         setFollowAgent(undefined);
@@ -110,8 +193,14 @@ const Sidebar: React.FC<SidebarProps> = (
     const renderControls = () => {
         return(
             <div className={styles.gameControls}>
-                <div>
+                <div className={styles.stepAndAudio}>
                     Step: {stepId}
+                    {process.env.NEXT_PUBLIC_ALLOW_AUDIO === "true" && <div>
+                        <label>
+                            <input type="checkbox" checked={isPlayAudio} onChange={() => setIsPlayAudio(!isPlayAudio)} />
+                            Play Audio
+                        </label>
+                    </div>}
                 </div>
                 <span style={{ display: 'none' }}>{substepId}</span>
                 <div className={styles.buttons}>