From 377cb5fc06b689e2387a7d0ff17072b10c4893e8 Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 1 Mar 2024 22:22:43 +0800 Subject: [PATCH 1/2] Load data in chunks and render simultaneously updated routes.ts and RenderLevel.tsx chunk size and interval can be fine tuned with further testing --- web/src/app/api/simulations/[id]/route.ts | 79 ++++++++++------ web/src/components/RenderLevel.tsx | 104 ++++++++++++---------- 2 files changed, 108 insertions(+), 75 deletions(-) diff --git a/web/src/app/api/simulations/[id]/route.ts b/web/src/app/api/simulations/[id]/route.ts index 4ebe4fa..66488fb 100644 --- a/web/src/app/api/simulations/[id]/route.ts +++ b/web/src/app/api/simulations/[id]/route.ts @@ -5,37 +5,62 @@ import { NextRequest } from "next/server"; // Initialize Redis connection const redis = getRedisClient(); +// Check if Redis client is connected +redis.on('connect', () => { + const redisDetails = `Connected to Redis server at ${redis.options.host}:${redis.options.port}`; + console.log(redisDetails); +}); + +// Check for errors during connection +redis.on('error', (err) => { + console.error('Error connecting to Redis server:', err); +}); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any -async function getStepsFromRedis(id: string, fromIndex: number): Promise<[number, any]> { +async function getStepsFromRedis(id: string, prevStartIndex: number, chunkSize: number): Promise<[number, any]> { try { const totalSteps = await redis.llen(id); - if(totalSteps - fromIndex <= 0){ + if (totalSteps - prevStartIndex <= 0) { + console.log("all steps have been fetched", prevStartIndex) return [totalSteps, []]; } - const stepsData = await redis.lrange(id, 0, totalSteps - fromIndex - 1); + var startIndex = 0; + if (totalSteps - prevStartIndex <= chunkSize) { + startIndex = -1; // last chunk, we start at index 0 + } + else { + startIndex = totalSteps - prevStartIndex - chunkSize; + } + + const stepsData = await redis.lrange(id, startIndex + 1, totalSteps - prevStartIndex); if (!stepsData) { throw new Error('No data found for steps'); } - return [totalSteps, stepsData]; + + return [totalSteps, stepsData]; + } catch (error) { console.error('Failed to fetch steps from Redis:', error); throw new Error('Failed to fetch steps data'); } } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any function uiSteps(step: any) { let keep = false; const keepTypes = ['talk', 'agent_set', 'move', 'matrix_set', 'agent_init']; - if(keepTypes.includes(step.step_type)) { + if (keepTypes.includes(step.step_type)) { keep = true; } - if(step.step_type === 'add_memory') { - if(step.kind === 'meta') { + if (step.step_type === 'add_memory') { + if (step.kind === 'meta') { keep = true; } } @@ -54,26 +79,26 @@ function sortSteps(a: any, b: any) { // Return a list of `params` to populate the [slug] dynamic segment export async function generateStaticParams() { - return [{id: 'skip'}]; + return [{ id: 'skip' }]; } export async function GET( - request: NextRequest, - { params }: { params: { id: string } } - - ) { - const id = params.id; - if (id === 'skip'){ - return Response.json([]); - } - - const fromIndex = Number(request.nextUrl.searchParams.get('fromIndex')) || 0; - - const [totalSteps, allSteps] = await getStepsFromRedis(id, fromIndex); - - let steps = allSteps.map((step: string) => JSON.parse(step)); - steps = steps.filter(uiSteps); - steps = steps.sort(sortSteps); - - return Response.json({ steps: steps, totalSteps: totalSteps }); -} \ No newline at end of file + request: NextRequest, + { params }: { params: { id: string } } +) { + const id = params.id; + if (id === 'skip') { + return Response.json([]); + } + + const fromIndex = Number(request.nextUrl.searchParams.get('fromIndex')) || 0; + const chunkSize = 500; // Adjust chunk size as needed + + const [totalSteps, allSteps] = await getStepsFromRedis(id, fromIndex, chunkSize); + + let steps = allSteps.map((step: string) => JSON.parse(step)); + steps = steps.filter(uiSteps); + steps = steps.sort(sortSteps); + + return Response.json({ steps: steps, totalSteps: totalSteps }); +} diff --git a/web/src/components/RenderLevel.tsx b/web/src/components/RenderLevel.tsx index 8b92ddf..d0a69bd 100644 --- a/web/src/components/RenderLevel.tsx +++ b/web/src/components/RenderLevel.tsx @@ -12,69 +12,77 @@ import { CELL_SIZE } from '@/helpers/consts'; async function getData(sim_id: string, fromIndex: number) { const redisKey = sim_id; const res = await fetch(`${process.env.NEXT_PUBLIC_API_URL}/${redisKey}?fromIndex=${fromIndex}`); - - + + if (!res.ok) { - // This will activate the closest `error.js` Error Boundary - throw new Error('Failed to fetch data') + // This will activate the closest `error.js` Error Boundary + throw new Error('Failed to fetch data') } - + const data = await res.json(); const steps = data['steps']; const totalSteps = data['totalSteps']; return [totalSteps, steps]; - } +} // eslint-disable-next-line @typescript-eslint/no-explicit-any -const RenderLevel: React.FC<{simId: string}> = ({ simId }) => { +const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { const [isPlaying, setIsPlaying] = useState(true); const [followAgent, setFollowAgent] = useState(undefined); - const [levelState, setLevelState] = useState({stepId: 0, substepId: 0, agents: []}); + const [levelState, setLevelState] = useState({ stepId: 0, substepId: 0, agents: [] }); + const [fetchIndex, setFetchIndex] = useState(0); + const [totalSteps, setTotalSteps] = useState(0); + const [initialFetchDone, setInitialFetchDone] = useState(false); + const chunkSize = 500; // Adjust chunk size as needed + const levelRef = useRef(new Level([], (newState: LevelState) => { setLevelState(newState); })); + const fetchData = async () => { + const [totalSteps, data] = await getData(simId, fetchIndex); + console.log(chunkSize); + setFetchIndex(fetchIndex + chunkSize); + + // Update totalSteps once + if (!initialFetchDone) { + setTotalSteps(totalSteps); + setInitialFetchDone(true); + } + + levelRef.current.addStepsFromJson(data); + + // if (fetchIndex >= totalSteps) { + // setIsPlaying(false); // or perform any other action when all data is fetched + // } + }; + useEffect(() => { - let fetchIndex = 0; let isMounted = true; - const level = levelRef.current; - - const fetchData = async () => { - if(!level.isPlaying) return; - - const [totalSteps, data] = await getData(simId, fetchIndex); - - if (fetchIndex < totalSteps && isMounted) { - levelRef.current.addStepsFromJson(data); - fetchIndex = totalSteps; - - if(level.timeline.dataComplete) { - clearInterval(interval); - } - } - }; - - fetchData(); - const interval = setInterval(fetchData, 12000); - - return () => { - isMounted = false; - levelRef.current.destroy(); - clearTimeout(interval); + if (!initialFetchDone) { + fetchData(); // Fetch immediately for the first time + } else { + const interval = setInterval(fetchData, 20000); // Subsequent fetches at 20-second intervals + + return () => { + isMounted = false; + clearInterval(interval); + }; } - }, [simId]); + }, [simId, fetchIndex]); + useEffect(() => { - if(levelState.agents.length > 0) { + if (levelState.agents.length > 0) { setFollowAgent(levelState.agents[0]); } }, [levelState.agents.length]); - - if(!levelState) { + + if (!levelState) { return (
Loading...
- ) + ) } const renderAgents = () => { @@ -89,10 +97,10 @@ const RenderLevel: React.FC<{simId: string}> = ({ simId }) => { }; return ( -
setFollowAgent(agent) }> +
setFollowAgent(agent)}>
); @@ -100,22 +108,22 @@ const RenderLevel: React.FC<{simId: string}> = ({ simId }) => { }; return ( - +
- Default Map + Default Map <> {renderAgents()} -
From e9baf58c121a7c4506b4d5ff8e23db24667123aa Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 1 Mar 2024 22:59:43 +0800 Subject: [PATCH 2/2] fixed compilation errors --- web/src/app/api/simulations/[id]/route.ts | 2 +- web/src/components/RenderLevel.tsx | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/web/src/app/api/simulations/[id]/route.ts b/web/src/app/api/simulations/[id]/route.ts index 66488fb..c8c6644 100644 --- a/web/src/app/api/simulations/[id]/route.ts +++ b/web/src/app/api/simulations/[id]/route.ts @@ -27,7 +27,7 @@ async function getStepsFromRedis(id: string, prevStartIndex: number, chunkSize: return [totalSteps, []]; } - var startIndex = 0; + let startIndex = 0; if (totalSteps - prevStartIndex <= chunkSize) { startIndex = -1; // last chunk, we start at index 0 } diff --git a/web/src/components/RenderLevel.tsx b/web/src/components/RenderLevel.tsx index d0a69bd..ae5bf0a 100644 --- a/web/src/components/RenderLevel.tsx +++ b/web/src/components/RenderLevel.tsx @@ -2,6 +2,7 @@ import Level, { LevelState } from '@/classes/Level'; import styles from './RenderLevel.module.css'; +import React from 'react'; import { useEffect, useRef, useState } from 'react'; import Sidebar from './Sidebar'; import AgentSprite from './AgentSprite'; @@ -32,7 +33,6 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { const [followAgent, setFollowAgent] = useState(undefined); const [levelState, setLevelState] = useState({ stepId: 0, substepId: 0, agents: [] }); const [fetchIndex, setFetchIndex] = useState(0); - const [totalSteps, setTotalSteps] = useState(0); const [initialFetchDone, setInitialFetchDone] = useState(false); const chunkSize = 500; // Adjust chunk size as needed @@ -42,12 +42,10 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { const fetchData = async () => { const [totalSteps, data] = await getData(simId, fetchIndex); - console.log(chunkSize); setFetchIndex(fetchIndex + chunkSize); + console.log(totalSteps); - // Update totalSteps once if (!initialFetchDone) { - setTotalSteps(totalSteps); setInitialFetchDone(true); } @@ -59,14 +57,14 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { }; useEffect(() => { - let isMounted = true; + // let isMounted = true; if (!initialFetchDone) { fetchData(); // Fetch immediately for the first time } else { const interval = setInterval(fetchData, 20000); // Subsequent fetches at 20-second intervals return () => { - isMounted = false; + // isMounted = false; clearInterval(interval); }; }