Skip to content

Commit

Permalink
Merge pull request #19 from chiefeu/main
Browse files Browse the repository at this point in the history 
Load data in chunks and render simultaneously
  • Loading branch information
@jtoy
jtoy authored Mar 3, 2024
2 parents ceba2d1 + e9baf58 commit 88fbc6b
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 76 deletions.
79 changes: 52 additions & 27 deletions web/src/app/api/simulations/[id]/route.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,62 @@ import { NextRequest } from "next/server";
// Initialize Redis connection
const redis = getRedisClient();

// Check if Redis client is connected
redis.on('connect', () => {
const redisDetails = `Connected to Redis server at ${redis.options.host}:${redis.options.port}`;
console.log(redisDetails);
});

// Check for errors during connection
redis.on('error', (err) => {
console.error('Error connecting to Redis server:', err);
});


// eslint-disable-next-line @typescript-eslint/no-explicit-any
async function getStepsFromRedis(id: string, fromIndex: number): Promise<[number, any]> {
async function getStepsFromRedis(id: string, prevStartIndex: number, chunkSize: number): Promise<[number, any]> {
try {
const totalSteps = await redis.llen(id);

if(totalSteps - fromIndex <= 0){
if (totalSteps - prevStartIndex <= 0) {
console.log("all steps have been fetched", prevStartIndex)
return [totalSteps, []];
}

const stepsData = await redis.lrange(id, 0, totalSteps - fromIndex - 1);
let startIndex = 0;
if (totalSteps - prevStartIndex <= chunkSize) {
startIndex = -1; // last chunk, we start at index 0
}
else {
startIndex = totalSteps - prevStartIndex - chunkSize;
}

const stepsData = await redis.lrange(id, startIndex + 1, totalSteps - prevStartIndex);
if (!stepsData) {
throw new Error('No data found for steps');
}
return [totalSteps, stepsData];

return [totalSteps, stepsData];

} catch (error) {
console.error('Failed to fetch steps from Redis:', error);
throw new Error('Failed to fetch steps data');
}
}



// eslint-disable-next-line @typescript-eslint/no-explicit-any
function uiSteps(step: any) {
let keep = false;
const keepTypes = ['talk', 'agent_set', 'move', 'matrix_set', 'agent_init'];

if(keepTypes.includes(step.step_type)) {
if (keepTypes.includes(step.step_type)) {
keep = true;
}

if(step.step_type === 'add_memory') {
if(step.kind === 'meta') {
if (step.step_type === 'add_memory') {
if (step.kind === 'meta') {
keep = true;
}
}
Expand All @@ -54,26 +79,26 @@ function sortSteps(a: any, b: any) {

// Return a list of `params` to populate the [slug] dynamic segment
export async function generateStaticParams() {
return [{id: 'skip'}];
return [{ id: 'skip' }];
}

export async function GET(
request: NextRequest,
{ params }: { params: { id: string } }
) {
const id = params.id;
if (id === 'skip'){
return Response.json([]);
}
const fromIndex = Number(request.nextUrl.searchParams.get('fromIndex')) || 0;

const [totalSteps, allSteps] = await getStepsFromRedis(id, fromIndex);
let steps = allSteps.map((step: string) => JSON.parse(step));
steps = steps.filter(uiSteps);
steps = steps.sort(sortSteps);
return Response.json({ steps: steps, totalSteps: totalSteps });
}
request: NextRequest,
{ params }: { params: { id: string } }
) {
const id = params.id;
if (id === 'skip') {
return Response.json([]);
}

const fromIndex = Number(request.nextUrl.searchParams.get('fromIndex')) || 0;
const chunkSize = 500; // Adjust chunk size as needed

const [totalSteps, allSteps] = await getStepsFromRedis(id, fromIndex, chunkSize);

let steps = allSteps.map((step: string) => JSON.parse(step));
steps = steps.filter(uiSteps);
steps = steps.sort(sortSteps);

return Response.json({ steps: steps, totalSteps: totalSteps });
}
104 changes: 55 additions & 49 deletions web/src/components/RenderLevel.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import Level, { LevelState } from '@/classes/Level';
import styles from './RenderLevel.module.css';
import React from 'react';
import { useEffect, useRef, useState } from 'react';
import Sidebar from './Sidebar';
import AgentSprite from './AgentSprite';
Expand All @@ -12,69 +13,74 @@ import { CELL_SIZE } from '@/helpers/consts';
async function getData(sim_id: string, fromIndex: number) {
const redisKey = sim_id;
const res = await fetch(`${process.env.NEXT_PUBLIC_API_URL}/${redisKey}?fromIndex=${fromIndex}`);


if (!res.ok) {
// This will activate the closest `error.js` Error Boundary
throw new Error('Failed to fetch data')
// This will activate the closest `error.js` Error Boundary
throw new Error('Failed to fetch data')
}

const data = await res.json();
const steps = data['steps'];
const totalSteps = data['totalSteps'];

return [totalSteps, steps];
}
}

// eslint-disable-next-line @typescript-eslint/no-explicit-any
const RenderLevel: React.FC<{simId: string}> = ({ simId }) => {
const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => {
const [isPlaying, setIsPlaying] = useState(true);
const [followAgent, setFollowAgent] = useState<Agent | undefined>(undefined);
const [levelState, setLevelState] = useState<LevelState>({stepId: 0, substepId: 0, agents: []});
const [levelState, setLevelState] = useState<LevelState>({ stepId: 0, substepId: 0, agents: [] });
const [fetchIndex, setFetchIndex] = useState(0);
const [initialFetchDone, setInitialFetchDone] = useState(false);
const chunkSize = 500; // Adjust chunk size as needed

const levelRef = useRef<Level>(new Level([], (newState: LevelState) => {
setLevelState(newState);
}));

const fetchData = async () => {
const [totalSteps, data] = await getData(simId, fetchIndex);
setFetchIndex(fetchIndex + chunkSize);
console.log(totalSteps);

if (!initialFetchDone) {
setInitialFetchDone(true);
}

levelRef.current.addStepsFromJson(data);

// if (fetchIndex >= totalSteps) {
// setIsPlaying(false); // or perform any other action when all data is fetched
// }
};

useEffect(() => {
let fetchIndex = 0;
let isMounted = true;
const level = levelRef.current;

const fetchData = async () => {
if(!level.isPlaying) return;

const [totalSteps, data] = await getData(simId, fetchIndex);

if (fetchIndex < totalSteps && isMounted) {
levelRef.current.addStepsFromJson(data);
fetchIndex = totalSteps;

if(level.timeline.dataComplete) {
clearInterval(interval);
}
}
};

fetchData();
const interval = setInterval(fetchData, 12000);

return () => {
isMounted = false;
levelRef.current.destroy();
clearTimeout(interval);
// let isMounted = true;
if (!initialFetchDone) {
fetchData(); // Fetch immediately for the first time
} else {
const interval = setInterval(fetchData, 20000); // Subsequent fetches at 20-second intervals

return () => {
// isMounted = false;
clearInterval(interval);
};
}
}, [simId]);
}, [simId, fetchIndex]);


useEffect(() => {
if(levelState.agents.length > 0) {
if (levelState.agents.length > 0) {
setFollowAgent(levelState.agents[0]);
}
}, [levelState.agents.length]);
if(!levelState) {

if (!levelState) {
return (
<div>Loading...</div>
)
)
}

const renderAgents = () => {
Expand All @@ -89,33 +95,33 @@ const RenderLevel: React.FC<{simId: string}> = ({ simId }) => {
};

return (
<div key={index}
style={style}
className={styles.placement}
onClick={() => setFollowAgent(agent) }>
<div key={index}
style={style}
className={styles.placement}
onClick={() => setFollowAgent(agent)}>
<AgentSprite agentName={agent.agentName} isTalking={agent.isTalking} isThinking={agent.isThinking} />
</div>
);
});
};

return (

<div className={styles.fullScreenContainer}>
<div className={styles.gameContainer}>

<Camera followAgent={followAgent} setFollowAgent={setFollowAgent}>
<img src={process.env.NEXT_PUBLIC_BASE_PATH +"/images/maps/Large.png"} alt="Default Map" />
<img src={process.env.NEXT_PUBLIC_BASE_PATH + "/images/maps/Large.png"} alt="Default Map" />
<>
{renderAgents()}
</>
</Camera>
<Sidebar agentPlacement={followAgent}
<Sidebar agentPlacement={followAgent}
setFollowAgent={setFollowAgent}
isPlaying={isPlaying}
isPlaying={isPlaying}
setIsPlaying={setIsPlaying}
stepId={levelState.stepId}
substepId={levelState.substepId}
stepId={levelState.stepId}
substepId={levelState.substepId}
level={levelRef.current} />
</div>
</div>
Expand Down

0 comments on commit 88fbc6b

Please sign in to comment.