diff --git a/README.md b/README.md index 2df2db6..78bcd2f 100644 --- a/README.md +++ b/README.md @@ -216,10 +216,18 @@ When working on the main engine, often times we can shut off all llm calls, in * make all cognitive modules flags work on a user basis * normalize flag names * hybrid fast llm action / continue to destination and only act when needed +* maybe make it gym compatible # changes over time + * moving objects +* tool usage + weapons + move refrigerator + door open/close/lock + +* scribblenauts style object interactions * information spreading * building stuff * people dead/people born @@ -231,5 +239,3 @@ When working on the main engine, often times we can shut off all llm calls, in * things getting destroyed * discover actions * environmental changes - - diff --git a/configs/bus_stop.json b/configs/bus_stop.json new file mode 100644 index 0000000..24b453b --- /dev/null +++ b/configs/bus_stop.json @@ -0,0 +1,87 @@ +{ + "background": "Six agents gather outside at a bus stop to chat with each other. They talk about their interests and anything they come up with.", + "actions_blacklist": [ "move" ], + "agents": [ + { + "name": "Natasha", + "description": "Natasha is friendly, outgoing, and effortlessly strikes up conversations. She likes to talk.", + "goal": "Natasha wants to talk about her recent experience with love. She wants to ask the opinions of other agents regarding the relationship. She is also concerned of the other agents and asks them if they are doing good. She talks about anything with the other agents..", + "connections": [ + "Sherlock", + "Viktor", + "Lily", + "James" + ], + "x": 25, + "y": 10 + }, + { + "name": "James", + "description": "James is shy, reserved, and observes from the sidelines.", + "goal": "James joins the conversation and shares his special skills and hidden talent in singing. He talks about his favorite songs and music genre. He also asks the other agents about their taste in music. He talks about anything with the other agents.", + "connections": [ + "Sherlock", + "Viktor", + "Lily", + "Natasha" + ], + "x": 25, + "y": 15 + }, + { + "name": "Sherlock", + "description": "Sherlock is funny, lighthearted, and turns moments into laughter. He is quick-witted and charming.", + "goal": "Sherlock tells a joke everytime he gets the chance to. He loves making everyone happy by being silly. He also shares some of his embarassing moments and also asks the other agents about their funny moments. He talks about anything with the other agents.", + "connections": [ + "Natasha", + "Viktor", + "James", + "Paul" + ], + "x": 25, + "y": 20 + }, + { + "name": "Viktor", + "description": "Viktor likes to take photographs and capture moments. He likes to share his collection of memories from pictures. He likes engaging in a conversation.", + "goal": "Viktor tells the other agents about his love of photography and capturing random sweet moments. He also willingly participates in every conversation with the other agents. He talks about anything with the other agents.", + "connections": [ + "James", + "Natasha", + "Sherlock", + "Paul" + ], + "x": 25, + "y": 25 + }, + { + "name": "Lily", + "description": "Lily likes to cook and eat delicious foods. She likes sweets the most. She is friendly and talkative.", + "goal": "Lily talks about her interest in cooking and eating sweet foods. She shares some of her favorite recipes and experience cooking them. She also asks the other agents about their favorite foods. She talks about anything with the other agents.", + "connections": [ + "Natasha", + "Viktor", + "Sherlock", + "Paul" + ], + "x": 25, + "y": 30 + }, + { + "name": "Paul", + "description": "Paul likes to read books especially poems. He is great at words and He is also quick witted.", + "goal": "Paul wants to tell the other agents about his favorite books and poems. He shares his some of the lines of poem he likes. He also tells a silly joke when he has a chance. He talks about anything with the other agents.", + "connections": [ + "Natasha", + "Viktor", + "James", + "Lily" + ], + "x": 25, + "y": 35 + } + ], + "steps": 50, + "allow_movement": 0, + "perception_range": 50 +} diff --git a/daemon.py b/daemon.py new file mode 100644 index 0000000..fbb4d6e --- /dev/null +++ b/daemon.py @@ -0,0 +1,65 @@ +import os +import redis +import json +import requests +from dotenv import load_dotenv +from engine import Matrix + +load_dotenv() + +REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379') +QUEUE_NAME = os.getenv('QUEUE_NAME', 'simulation_jobs') +DISCORD_URL = os.getenv('DISCORD_URL') + +# Connect to Redis +redis_conn = redis.from_url(REDIS_URL) + +def notify_discord(msg): + if DISCORD_URL: + requests.post(DISCORD_URL,json={'content':msg}) + +# get this out of here or refactor with engine +def update_from_env(config): + for key, value in config.items(): + env_var = os.getenv(key) + if env_var is not None: + # Update the value from the environment variable + config[key] = type(value)(env_var) if value is not None else env_var + return config + +def load_config(): + filename = "configs/defaults.json" + with open(filename, 'r') as file: + config = json.load(file) + config = update_from_env(config) + return config + +def process_simulation(data): + config = load_config() + config['scenario'] = data + config['environment'] = "configs/largev2.tmj" + notify_discord(f"starting simulation: #{config}") + matrix = Matrix(config) + matrix.boot() + matrix.run_singlethread() + notify_discord(f"finished simulation: #{config}") + + + print(f'Simulation {simulation_id} completed.') + +def main(): + print('Starting simulation job daemon...') + while True: + # Fetch a job from the Redis queue + _, job = redis_conn.blpop(QUEUE_NAME) + job_data = json.loads(job) + + # Process the simulation + try: + process_simulation(job_data) + except Exception as e: + print(f'Error processing simulation: {e}') + +if __name__ == '__main__': + main() + diff --git a/engine.py b/engine.py index 28c8590..0a00f82 100755 --- a/engine.py +++ b/engine.py @@ -46,15 +46,8 @@ def main(): if args.id: config['id'] = args.id matrix = Matrix(config) - - # matrix.send_matrix_to_redis() - pd(f"model:#{MODEL}") - pd("Initial Agents Positions:") - #redis_connection.set(f"{matrix.id}:matrix_state", json.dumps(matrix.get_arr_2D())) - # Clear convos - #matrix.clear_redis() # Run start_time = datetime.now() @@ -67,7 +60,6 @@ def main(): # Log Runtime matrix.simulation_runtime = end_time - start_time - matrix.send_matrix_to_redis() # Save the environment state to a file for inspection if matrix.id is not None and matrix.id != '' and RUN_REPORTS != 0: @@ -91,7 +83,6 @@ def signal_handler(signum, frame): pd("stopping matrix, please wait for current step to finish") pd("*"*50) matrix.status = "stop" - matrix.send_matrix_to_redis() last_interrupt_time = current_time ctrl_c_count = 0 diff --git a/frontend.py b/frontend.py deleted file mode 100644 index a84d0a5..0000000 --- a/frontend.py +++ /dev/null @@ -1,99 +0,0 @@ -from flask import Flask, request, jsonify, render_template -from redis import Redis -import requests - -app = Flask(__name__) -redis_client = Redis.from_url("redis://localhost:6379") - -@app.route('/get_matrix_state') -def get_matrix_state(): - try: - matrix_state = redis_client.get(':matrix_state') - - matrix = matrix_state.decode('utf-8') if matrix_state else '[]' - - return jsonify(matrix) - - except Exception as e: - return jsonify({'error': str(e)}), 500 - -@app.route('/get_conversations') -def get_conversations(): - try: - # Fetch conversations from Redis queue "agent_conversations" - conversations = redis_client.lrange(':agent_conversations', 0, -1)[::-1] - - # Convert byte strings to regular strings - conversations = [conversation.decode('utf-8') for conversation in conversations] - - return jsonify(conversations) - except Exception as e: - return jsonify({'error': str(e)}), 500 - -@app.route('/') -def index(): - return render_template('frontend.html') - -@app.route('/map') -def map(): - return render_template('map.html') - -@app.route('/embeddings', methods=['POST']) -def embeddings(): - # Get data from the request - data = request.json - - # Extract values from the payload - model = data.get('model') - prompt = data.get('prompt') - - # Create the payload for the POST request to the external API - payload = { - "model": model, - "prompt": prompt - } - - # Make the POST request to the external API - external_api_url = "http://localhost:11434/api/embeddings" - response = requests.post(external_api_url, json=payload) - - # Check if the request was successful (status code 200) - if response.status_code == 200: - # Parse the JSON response from the external API - result = response.json() - return jsonify(result) - else: - # If the request was not successful, return an error message - return jsonify({"error": f"Failed to generate data. Status code: {response.status_code}"}), response.status_code - -@app.route('/generate', methods=['POST']) -def generate(): - # Get data from the request - data = request.json - - # Extract values from the payload - model = data.get('model') - prompt = data.get('prompt') - - # Create the payload for the POST request to the external API - payload = { - "model": model, - "prompt": prompt, - "stream": False - } - - # Make the POST request to the external API - external_api_url = "http://localhost:11434/api/generate" - response = requests.post(external_api_url, json=payload) - - # Check if the request was successful (status code 200) - if response.status_code == 200: - # Parse the JSON response from the external API - result = response.json() - return jsonify(result) - else: - # If the request was not successful, return an error message - return jsonify({"error": f"Failed to generate data. Status code: {response.status_code}"}), response.status_code - -if __name__ == '__main__': - app.run(debug=True,host='0.0.0.0') diff --git a/prompts/general/summarize_conversation.txt b/prompts/general/summarize_conversation.txt index 8063a9a..4692d8b 100644 --- a/prompts/general/summarize_conversation.txt +++ b/prompts/general/summarize_conversation.txt @@ -1,7 +1,7 @@ Your name is {{agent}} -You are talking with {{other_agent}} +You just spoke with {{other_agent}} {{conversation_string}} -Summarize the conversation above in {{agent}}'s point of view. -List all important outcomes of the conversation: +Summarize the conversation in the first person point of view as {{agent}}. +Write it as a thought {{agent}} says to theirselves. diff --git a/prompts/general/talk.txt b/prompts/general/talk.txt index d557348..909d14d 100644 --- a/prompts/general/talk.txt +++ b/prompts/general/talk.txt @@ -1,18 +1,14 @@ -roleplay as {{agent}} -{{selfContext}} -{{relevant_memories}} +roleplay as {{agent}} talking to {% if other_agent == "stranger" %} a stranger you know nothing about. {% else %} {{other_agent}} {% endif %} -{% if other_agent == "stranger" %} -This is your first time talking to this stranger. +{{selfContext}} +{{agent}}'s previous memories: +{{relevant_memories}} -{% else %} -{% if primer %} +{% if other_agent != "stranger" and primer %} prime your thoughts but dont mention: "{{primer}}". -{% endif %} -{{selfContext}} Below is the current chat history between {{agent}} and {{other_agent}}. {{previous_conversations}} @@ -41,5 +37,6 @@ Maybe you could discuss the topic of "{{topic}}" keep responses to under 3 sentences. Craft an informal spoken response. +Do not discuss topics for things that you don't perceive or are not in your recent memories. Only write the response from {{agent}} and nothing else. diff --git a/requirements.txt b/requirements.txt index 0c7378e..9429723 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,6 @@ python-Levenshtein jsonlines nltk Pillow +groq +sshtunnel +psycopg2 diff --git a/src/agents.py b/src/agents.py index bed7409..e9ae121 100644 --- a/src/agents.py +++ b/src/agents.py @@ -32,9 +32,8 @@ def __init__(self, agent_data={}): self.x = agent_data.get("x", None) self.y = agent_data.get("y", None) self.actions = list(set(agent_data.get("actions", []) + DEFAULT_ACTIONS)) - #self.actions = list(dict.fromkeys(agent_data.get("actions", []) + DEFAULT_ACTIONS)) - #old one - self.actions = list(dict.fromkeys(agent_data.get("actions",DEFAULT_ACTIONS ))) + self.last_perceived_data = None + self.perceived_data_history = {} self.memory = agent_data.get("memory", []) self.short_memory = agent_data.get("short_memory", []) @@ -62,8 +61,29 @@ def __init__(self, agent_data={}): self.matrix = agent_data.get('matrix') if self.matrix: + if self.matrix.action_blacklist: + self.actions = [action for action in self.actions if action not in blacklist] + if self.matrix and self.matrix.environment: + valid_coordinates = self.matrix.environment.get_valid_coordinates() + if (self.x, self.y) not in valid_coordinates: + new_position = random.choice(valid_coordinates) + self.x = new_position[0] + self.y = new_position[1] self.matrix.add_to_logs({"agent_id":self.mid,"step_type":"agent_init","x":self.x,"y":self.y,"name":self.name,"goal":self.goal,"kind":self.kind,"description":self.description,"status":self.status}) + def perceived_data_is_same(self): + last_step = self.matrix.cur_step - 1 + if last_step not in self.perceived_data_history: + return False + perceived_agents, perceived_locations, perceived_areas, perceived_objects,perceived_directions = self.perceive([a for a in self.matrix.agents if a != self], self.matrix.environment, unix_to_strftime(self.matrix.unix_time)) + current_perceived_data = (perceived_agents, perceived_locations, perceived_areas, perceived_objects, perceived_directions) + return self.perceived_data_history[last_step] == current_perceived_data + + def cleanup_old_perceived_data(self, current_step): + keys_to_delete = [step for step in self.perceived_data_history if step < current_step - 2] + for key in keys_to_delete: + del self.perceived_data_history[key] + def update_goals(self): # this assumes 8+ importance is always worth changing /reevaluating goals # do i need to update my goals. if so, give me new goals @@ -84,6 +104,7 @@ def update_goals(self): self.addMemory("observation",f"I updated my goal to be \"{msg}\"", unix_to_strftime(self.matrix.unix_time), random.randint(5, 8)) self.goal = msg + def decide(self): self.matrix.llm_action(self,self.matrix.unix_time) @@ -97,7 +118,11 @@ def ask_meta_questions(self, timestamp): self.meta_questions.extend(x[1] for x in m if x[1] not in self.meta_questions) def evaluate_progress(self,opts={}): - relevant_memories = self.getMemories(self.goal, unix_to_strftime(self.matrix.unix_time)) + if self.matrix: + relevant_memories = self.getMemories(self.goal, unix_to_strftime(self.matrix.unix_time)) + else: + relevant_memories = self.getMemories(self.goal, opts.get("timestamp", "")) + relevant_memories_string = "\n".join(f"Memory {i + 1}:\n{memory}" for i, memory in enumerate(relevant_memories)) if relevant_memories else "" primer = opts.get("random_prime",False) variables = { @@ -114,7 +139,11 @@ def evaluate_progress(self,opts={}): score = int(match.group(1)) if match else None if score and explanation: - self.addMemory("meta", explanation, unix_to_strftime(self.matrix.unix_time) , 10) + if self.matrix: + self.addMemory("meta", explanation, unix_to_strftime(self.matrix.unix_time) , 10) + else: + self.addMemory("meta", explanation, opts.get("timestamp", "") , 10) + if score and int(score) < 3: #self.meta_cognize(unix_to_strftime(self.matrix.unix_time), True) pass @@ -284,10 +313,11 @@ def summarize_conversation(self, timestamp): msg = llm.prompt(prompt_name="summarize_conversation", variables=variables) interaction = f"{timestamp} - {self} summarized their conversation with {self.last_conversation.other_agent.name}: {msg}" + interaction = f"my conversation with {self.last_conversation.other_agent.name}: {msg}" if self.matrix is not None: print_and_log(interaction, f"{self.matrix.id}:events:{self.name}") - self.addMemory("conversation", interaction, timestamp, random.randint(4, 6)) + self.addMemory("summary", interaction, timestamp, random.randint(4, 6)) if self.matrix: self.matrix.add_to_logs({"step_type":"agent_set", "attribute_name": "convo", "attribute_data": {"status": "complete", "from":self.mid, "to":self.last_conversation.other_agent.mid, "convo_id":self.last_conversation.mid}}) self.last_conversation = None @@ -477,7 +507,7 @@ def perceive(self, other_agents, environment, timestamp): perceived_agents = [] perceived_areas = [] perceived_directions = [] - if (self.matrix is not None and self.matrix.allow_observance_flag == 0) or (self.matrix is None and ALLOW_OBSERVE == 0): + if (self.matrix is not None and self.matrix.allow_observance_flag == 0) or (self.matrix is None and ALLOW_OBSERVANCE == 0): return perceived_agents, perceived_locations, perceived_areas, perceived_objects @@ -564,7 +594,13 @@ def perceive(self, other_agents, environment, timestamp): perceived_agent_ids = [agent.mid for agent in perceived_agents] if self.matrix: self.matrix.add_to_logs({"agent_id":self.mid,"step_type":"perceived","perceived_agents":perceived_agent_ids,"perceived_locations":[],"perceived_areas":[],"perceived_objects":[]}) - #missing locations,areas,objects + + perceived_data = (perceived_agents, perceived_locations, perceived_areas, perceived_objects, perceived_directions) + #self.last_perceived_data = perceived_data + if self.matrix: + self.perceived_data_history[self.matrix.cur_step] = perceived_data + self.cleanup_old_perceived_data(self.matrix.cur_step) + return perceived_agents, perceived_locations, perceived_areas, perceived_objects,perceived_directions def addMemory(self, kind, content, timestamp=None, score=None,embedding=None): @@ -574,7 +610,7 @@ def addMemory(self, kind, content, timestamp=None, score=None,embedding=None): if kind == "observation": - if (self.matrix is not None and self.matrix.allow_observance_flag == 1): + if (self.matrix and self.matrix.allow_observance_flag == 1) or (self.matrix is None): memory = Memory(kind, content, timestamp, timestamp, score,embedding) self.memory.append(memory) else: diff --git a/src/data.py b/src/data.py new file mode 100644 index 0000000..b7ce219 --- /dev/null +++ b/src/data.py @@ -0,0 +1,100 @@ +import os +import json +import jsonlines +import sys +import psycopg2 +from dotenv import load_dotenv +from sshtunnel import SSHTunnelForwarder +import redis +load_dotenv() + +class Data: + def setup_redis(self): + if os.environ.get("REDIS_URL"): + return redis.Redis.from_url(os.environ.get("REDIS_URL")) + else: + return None + def setup_database(self): + db_settings = { + "database_host": os.environ.get("DB_HOST"), + "database_port": int(os.environ.get("DB_PORT")), + "database_name": os.environ.get("DB_NAME"), + "database_username": os.environ.get("DB_USER"), + "database_password": os.environ.get("DB_PASSWORD"), + } + + # SSH settings + ssh_host = os.environ.get("SSH_HOST") + ssh_settings = { + "ssh_host": ssh_host, + "ssh_port": int(os.environ.get("SSH_PORT", 22)), + "ssh_user": os.environ.get("SSH_USER"), + "ssh_private_key": os.environ.get("SSH_PRIVATE_KEY") + } + cursor = None + conn = None + if ssh_host: + with SSHTunnelForwarder( + (ssh_settings["ssh_host"], ssh_settings["ssh_port"]), + ssh_username=ssh_settings["ssh_user"], + ssh_pkey=ssh_settings["ssh_private_key"], + remote_bind_address=(db_settings["database_host"], db_settings["database_port"]), + ) as tunnel: + # Establish database connection + conn = psycopg2.connect( + user=db_settings["database_username"], + password=db_settings["database_password"], + host=db_settings["database_host"], + port=tunnel.local_bind_port, + database=db_settings["database_name"], + ) + cursor = conn.cursor() + elif db_settings["database_host"]: + conn = psycopg2.connect( + user=db_settings["database_username"], + password=db_settings["database_password"], + host=db_settings["database_host"], + port=db_settings["database_port"], + database=db_settings["database_name"], + ) + + if conn: + return conn.cursor() + else: + return None + + def add_to_logs(self,obj): + obj["step"] = self.cur_step + obj["substep"] = self.current_substep + obj["sim_id"] = self.id # i think we will change this to sim id everywhere + + file = f"logs/{obj['sim_id']}.jsonl" + #with open("logs.json", "a") as file: + # json.dump(obj,file,indent=2) + # file.write("\n\n") + with jsonlines.open(file, mode='a') as writer: + writer.write(json.dumps(obj)) + stream = f"{obj['sim_id']}_stream" + queue = f"{obj['sim_id']}" + wtf = json.loads(json.dumps(obj, default=str)) + #redis_connection.xadd(stream, wtf) + max_retries = 3 + retry_delay = 1 + if self.redis_connection: + for attempt in range(max_retries): + try: + self.redis_connection.lpush(queue, json.dumps(obj)) + break # Break the loop if successful + except redis.RedisError as e: + print(f"Error pushing to Redis queue. Retrying... ({attempt + 1}/{max_retries})") + time.sleep(retry_delay) + + if self.cursor and obj["step_type"] in ['talk', 'agent_set', 'move', 'matrix_set', 'agent_init']: + fields_to_skip = ["step", "substep", "step_type","sim_id","embedding"] + data = {k: v for k, v in obj.items() if k not in fields_to_skip} + self.cursor.execute( + "INSERT INTO timelines (sim_id, step, substep, step_type, data) VALUES (%s, %s, %s, %s, %s)", + (self.id, self.cur_step, self.current_substep, obj["step_type"], json.dumps(obj)),) + #print("insert") + + self.current_substep += 1 diff --git a/src/matrix.py b/src/matrix.py index 058cc81..75d8e7d 100644 --- a/src/matrix.py +++ b/src/matrix.py @@ -21,13 +21,15 @@ from src.npc import Npc from src.location import Location, Area, Object from src.actions.fine_move_action import FineMoveAction +from src.data import Data +from src.reporting import Reporting def set_globals(config): for key, value in config.items(): globals()[key] = value -class Matrix: +class Matrix(Data,Reporting): def __init__(self, config={}): set_globals(config) self.steps = SIMULATION_STEPS @@ -69,13 +71,15 @@ def __init__(self, config={}): self.perception_range = PERCEPTION_RANGE self.allow_movement = ALLOW_MOVEMENT self.model = MODEL + self.redis_connection = self.setup_redis() self.replay = None + self.cursor = self.setup_database() self.add_to_logs({"step_type":"matrix_init","data":config}) self.agent_locks = { agent: threading.Lock() for agent in self.agents } self.environment = Environment({ "filename": self.environment_file }) if self.scenario_file is not None: - self.parse_scenario_file(self.scenario_file) + self.parse_scenario(self.scenario_file) #self.environment.overlay_collisions_on_image() self.background = None print(config) @@ -100,7 +104,7 @@ def boot(self): # Add Zombies for i in range(self.num_zombies): - zombie = Agent({ "name": f"Zombie_{i}", "kind": "zombie", "actions": ["kill"],"matrix":self }) + zombie = Agent({ "name": f"killer Zombie {i}", "kind": "zombie", "actions": ["kill"],"matrix":self }) self.add_agent_to_simulation(zombie) @classmethod @@ -162,12 +166,13 @@ def from_timeline(cls,src,step=None): return(matrix) - - - - def parse_scenario_file(self, filename): - with open(filename, 'r') as file: - data = json.load(file) + def parse_scenario(self, config): + if isinstance(config, str): + with open(config, 'r') as file: + data = json.load(file) + self.data = data + else: + data = config self.data = data # Build Scenario @@ -175,8 +180,14 @@ def parse_scenario_file(self, filename): self.allow_movement = data.get("allow_movement", ALLOW_MOVEMENT) self.background = data.get("background", "") self.performance_evals = data.get("performance", {}) - self.performance_metrics[self.performance_evals["numerator"]] = 0 - self.performance_metrics["denominator"] = self.performance_evals["denominator"] + if not self.performance_evals: + self.performance_metrics["total_alive"] = 0 + self.performance_metrics["denominator"] = "total_agents" + else: + self.performance_metrics[self.performance_evals["numerator"]] = 0 + self.performance_metrics["denominator"] = self.performance_evals["denominator"] + + self.action_blacklist = data.get("action_blacklist",[]) if self.steps <= 0: self.steps = data.get("steps", 100) @@ -224,116 +235,12 @@ def add_agent_to_simulation(self, agent): self.agents.append(agent) - def add_to_logs(self,obj): - obj["step"] = self.cur_step - obj["substep"] = self.current_substep - obj["sim_id"] = self.id # i think we will change this to sim id everywhere - - file = f"logs/{obj['sim_id']}.jsonl" - #with open("logs.json", "a") as file: - # json.dump(obj,file,indent=2) - # file.write("\n\n") - with jsonlines.open(file, mode='a') as writer: - writer.write(json.dumps(obj)) - stream = f"{obj['sim_id']}_stream" - queue = f"{obj['sim_id']}" - wtf = json.loads(json.dumps(obj, default=str)) - #redis_connection.xadd(stream, wtf) - max_retries = 3 - retry_delay = 1 - if redis_connection: - for attempt in range(max_retries): - try: - redis_connection.lpush(queue, json.dumps(obj)) - break # Break the loop if successful - except redis.RedisError as e: - print(f"Error pushing to Redis queue. Retrying... ({attempt + 1}/{max_retries})") - time.sleep(retry_delay) - - self.current_substep += 1 - - def get_server_info(self): - try: - # Run 'uname -a' command - uname_output = subprocess.check_output(['uname', '-a']).decode('utf-8').strip() - return uname_output - except Exception as e: - # Handle any exceptions that may occur - return f"Error getting server info: {str(e)}" - - def all_env_vars(self): - if self.sim_start_time is None: - self.sim_start_time = datetime.now() - - if self.simulation_runtime is None: - self.simulation_runtime = datetime.now() - self.sim_start_time - - total_reflections = 0 - total_metas = 0 - for a in self.agents: - for m in a.memory: - if m.kind == "reflect": - total_reflections += 1 - if m.kind == "meta": - total_metas += 1 - - total_seconds = self.simulation_runtime.total_seconds() - - # Calculate minutes and seconds - minutes = int(total_seconds // 60) - seconds = int(total_seconds % 60) - - # Create a human-readable string - runtime_string = f"{minutes} minute(s) and {seconds} second(s)" - - return { - "id": self.id, - "map": self.environment_file, - "agents": self.scenario_file, - "date": self.sim_start_time.isoformat(), - "width": self.environment.width, - "height": self.environment.width, - "status": self.status, - "runtime": runtime_string, # Include the string representation - "server_info": self.get_server_info(), - "created_at": self.sim_start_time.strftime("%Y-%m-%d %H:%M:%S"), - "model": self.model, - "total_steps": self.steps, - "meta_flag": self.allow_meta_flag, - "reflect_flag": self.allow_reflect_flag, - "conversation_counter": self.conversation_counter, - "total_meta_memories": total_metas, - "total_reflect_memories": total_reflections, - "total_agents": sum(1 for agent in self.agents if agent.kind != 'zombie'), - "total_zombies": sum(1 for agent in self.agents if agent.kind == 'zombie'), - "total_dead": sum(1 for agent in self.agents if agent.status == 'dead'), - "total_alive": sum(1 for agent in self.agents if agent.status != 'dead'), - "llm_call_counter": llm.call_counter, - "avg_runtime_per_step": total_seconds / self.steps, - "avg_llm_calls_per_step": llm.call_counter / self.steps - } - - def send_matrix_to_redis(self): - if TEST_RUN == 0: - #redis_connection.set(f"{self.id}:simulations", json.dumps(self.all_env_vars())) - pass - - def log_agents_to_redis(self): - for agent in self.agents: - agent_data = { - "name": agent.name, - "x": agent.x, - "y": agent.y, - "status": agent.status - } - #redis_connection.rpush(f"{self.id}:agents:{agent.name}", json.dumps(agent_data)) def run_singlethread(self): #self.boot() self.status = "running" self.sim_start_time = datetime.now() - #self.send_matrix_to_redis() for step in range(self.steps): self.cur_step = step self.current_substep = 0 @@ -344,16 +251,6 @@ def run_singlethread(self): start_time = datetime.now() pd(f"Step {step + 1}:") - #redis_log(self.get_arr_2D(), f"{self.id}:matrix_states") - #redis_connection.set(f"{self.id}:matrix_state", json.dumps(self.get_arr_2D())) - #print_and_log(f"Step: {step + 1} | {unix_to_strftime(self.unix_time)}", f"{self.id}:agent_conversations") - - #self.log_agents_to_redis() - - #for a in self.agents: - # print_and_log(f"Step: {step + 1} | {unix_to_strftime(self.unix_time)}", f"{self.id}:events:{a.name}") - # print_and_log(f"Step: {step + 1} | {unix_to_strftime(self.unix_time)}", f"{self.id}:conversations:{a.name}") - if redis_connection: control_cmd = redis_connection.lpop(f"{self.id}:communications") if control_cmd: @@ -451,6 +348,13 @@ def llm_action(self, agent, unix_time): if agent.status == "dead": return agent + perceived_agents, perceived_locations, perceived_areas, perceived_objects,perceived_directions = agent.perceive([a for a in self.agents if a != agent], self.environment, unix_to_strftime(self.unix_time)) + + if agent.current_destination is not None and agent.perceived_data_is_same(): + print("SKIPPED llm_action!") + agent.move({ "environment": self.environment }) + return agent + # It is 12:00, time to make plans if unix_time % 86400 == 0 and self.allow_plan_flag == 1: agent.make_plans(unix_to_strftime(unix_time)) @@ -482,7 +386,6 @@ def llm_action(self, agent, unix_time): agent.talk({ "other_agents": [agent.last_conversation.other_agent], "timestamp": unix_to_strftime(unix_time) }) return agent - perceived_agents, perceived_locations, perceived_areas, perceived_objects,perceived_directions = agent.perceive([a for a in self.agents if a != agent], self.environment, unix_to_strftime(self.unix_time)) relevant_memories = agent.getMemories(agent.goal, unix_to_strftime(unix_time)) relevant_memories_string = "\n".join(f"Memory {i + 1}:\n{memory}" for i, memory in enumerate(relevant_memories)) if relevant_memories else "" @@ -652,11 +555,6 @@ def agent_action(self, agent, unix_time): return agent # Agent stays in the same position - def print_agent_memories(self): - for agent in self.agents: - pd(f"\nMemories for {agent}:") - for memory in agent.memory: - pd(memory) @@ -673,75 +571,3 @@ def generate_unique_name(self): new_name = f"Agent{len(self.agents) + 1}" print(f"New Name: {new_name}") return new_name - - def run_interviews(self): - if self.interview_questions: - dead_agents = [agent for agent in self.agents if (agent.status == "dead" and agent.kind != "zombie")] - living_agents = [agent for agent in self.agents if (agent.status != "dead" and agent.kind != "zombie")] - - for agent in dead_agents + living_agents: - results = [] - for question in self.interview_questions: - metric = question.get("metric", None) - #if agent.status == "dead": - # pd(f"{agent} dead, can't ask questions") - # results.append("Agent is dead, cannot answer questions") - #elif question["who"] == "all" or question["who"] == agent.name: - answer = agent.answer(question["question"]) - if metric: - match = re.search(r"Answer: (\d)", answer) - if match: - score = int(match.group(0)) - else: - score = 0 - self.performance_metrics[metric] += score - answer_data = { - "question": question["question"], - "answer": answer - } - results.append(answer_data) - self.interview_results[agent.name] = results - - def print_matrix(self): - cell_width = 15 # Adjust this value based on your needs - matrix = [[" " * cell_width for _ in range(self.environment.width)] for _ in range(self.environment.height)] - - # Print agents - for agent in self.agents: - matrix[agent.x][agent.y] = "{:<{width}}".format(f"{agent.direction} * {agent.name}", width=cell_width) - - #sys.stdout.write("\033[H") - print("\n\n") - for row in matrix: - print("|".join(row)) - print("-" * (cell_width * self.n - 1)) - - def get_all_objects(self): - all_objects = [obj for loc in self.locations for area in loc.areas for obj in area.objects] - return all_objects - - def get_arr_2D(self): - arr_2D = [["" for _ in range(self.environment.width)] for _ in range(self.environment.height)] - objs = [obj for location in self.environment.locations for area in location.areas for obj in area.objects] - - for x in range(self.environment.height): - for y in range(self.environment.width): - for obj in objs: - if obj.bounds[x][y] != 0: - arr_2D[x][y] = obj.name[0].lower() - - for agent in self.agents: - arr_2D[agent.x][agent.y] = f"{agent}" - - return arr_2D - - def clear_redis(self): - return - redis_connection.delete(f"{self.id}:matrix_state") - redis_connection.delete(f"{self.id}:matrix_states") - redis_connection.delete(f"{self.id}:agent_conversations") - for a in self.agents: - redis_connection.delete(f"{self.id}:conversations:{a.name}") - redis_connection.delete(f"{self.id}:events:{a.name}") - redis_connection.delete(f"{self.id}:agents:{a.name}") - diff --git a/src/reporting.py b/src/reporting.py new file mode 100644 index 0000000..2bfa314 --- /dev/null +++ b/src/reporting.py @@ -0,0 +1,112 @@ +from dotenv import load_dotenv +import subprocess +from utils.utils import * + +class Reporting: + def get_server_info(self): + try: + # Run 'uname -a' command + uname_output = subprocess.check_output(['uname', '-a']).decode('utf-8').strip() + return uname_output + except Exception as e: + # Handle any exceptions that may occur + return f"Error getting server info: {str(e)}" + + def all_env_vars(self): + if self.sim_start_time is None: + self.sim_start_time = datetime.now() + + if self.simulation_runtime is None: + self.simulation_runtime = datetime.now() - self.sim_start_time + + total_reflections = 0 + total_metas = 0 + for a in self.agents: + for m in a.memory: + if m.kind == "reflect": + total_reflections += 1 + if m.kind == "meta": + total_metas += 1 + + total_seconds = self.simulation_runtime.total_seconds() + + # Calculate minutes and seconds + minutes = int(total_seconds // 60) + seconds = int(total_seconds % 60) + + # Create a human-readable string + runtime_string = f"{minutes} minute(s) and {seconds} second(s)" + + return { + "id": self.id, + "map": self.environment_file, + "agents": self.scenario_file, + "date": self.sim_start_time.isoformat(), + "width": self.environment.width, + "height": self.environment.width, + "status": self.status, + "runtime": runtime_string, # Include the string representation + "server_info": self.get_server_info(), + "created_at": self.sim_start_time.strftime("%Y-%m-%d %H:%M:%S"), + "model": self.model, + "total_steps": self.steps, + "meta_flag": self.allow_meta_flag, + "reflect_flag": self.allow_reflect_flag, + "conversation_counter": self.conversation_counter, + "total_meta_memories": total_metas, + "total_reflect_memories": total_reflections, + "total_agents": sum(1 for agent in self.agents if agent.kind != 'zombie'), + "total_zombies": sum(1 for agent in self.agents if agent.kind == 'zombie'), + "total_dead": sum(1 for agent in self.agents if agent.status == 'dead'), + "total_alive": sum(1 for agent in self.agents if agent.status != 'dead'), + "llm_call_counter": llm.call_counter, + "avg_llm_calls_per_step": llm.call_counter / self.steps, + "avg_runtime_per_step": total_seconds / self.steps, + } + def run_interviews(self): + if self.interview_questions: + dead_agents = [agent for agent in self.agents if (agent.status == "dead" and agent.kind != "zombie")] + living_agents = [agent for agent in self.agents if (agent.status != "dead" and agent.kind != "zombie")] + + for agent in dead_agents + living_agents: + results = [] + for question in self.interview_questions: + metric = question.get("metric", None) + #if agent.status == "dead": + # pd(f"{agent} dead, can't ask questions") + # results.append("Agent is dead, cannot answer questions") + #elif question["who"] == "all" or question["who"] == agent.name: + answer = agent.answer(question["question"]) + if metric: + match = re.search(r"Answer: (\d)", answer) + if match: + score = int(match.group(0)) + else: + score = 0 + self.performance_metrics[metric] += score + answer_data = { + "question": question["question"], + "answer": answer + } + results.append(answer_data) + self.interview_results[agent.name] = results + + def print_agent_memories(self): + for agent in self.agents: + print(f"\nMemories for {agent}:") + for memory in agent.memory: + print(memory) + + def print_matrix(self): + cell_width = 15 # Adjust this value based on your needs + matrix = [[" " * cell_width for _ in range(self.environment.width)] for _ in range(self.environment.height)] + + # Print agents + for agent in self.agents: + matrix[agent.x][agent.y] = "{:<{width}}".format(f"{agent.direction} * {agent.name}", width=cell_width) + + #sys.stdout.write("\033[H") + print("\n\n") + for row in matrix: + print("|".join(row)) + print("-" * (cell_width * self.n - 1)) diff --git a/test.py b/test.py index 3b612ba..f2129a7 100644 --- a/test.py +++ b/test.py @@ -16,6 +16,11 @@ from src.actions.fine_move_action import FineMoveAction class TestMemoryFunctions(unittest.TestCase): + def test_web(self): + #run_simulation("steps":1) + #start_npm_run_dev + #assert_check_for_event + pass def test_fine_move_action_direction(self): agent = MagicMock(x=0, y=0) FineMoveAction.act(agent, "up with random text") @@ -29,14 +34,37 @@ def test_fine_move_action_invalid_direction(self): self.assertEqual(agent.y, 0) def test_fine_move_action_moves_away(self): - #real agent here, real zombie here. decide - matrix = Matrix({"environment":"configs/small.tmj"}) - real_agent = Agent({ "name": "John", "actions": ["fine_move"],"x":5,"y":5,"matrix":matrix }) - matrix.add_agent_to_simulation(real_agent) - zombie = Agent({ "name": f"Zombie", "kind": "zombie", "actions": ["kill"],"x":6,"y":5,"matrix":matrix }) - matrix.add_agent_to_simulation(zombie) - matrix.llm_action(real_agent, matrix.unix_time) - self.assertEqual(real_agent.x, 4) + successful_outcomes = 0 + for _ in range(4): + matrix = Matrix({"environment": "configs/small.tmj"}) + real_agent = Agent({"name": "John", "actions": ["fine_move"], "x": 5, "y": 5, "matrix": matrix}) + matrix.add_agent_to_simulation(real_agent) + zombie = Agent({"name": f"killer Zombie", "kind": "zombie", "actions": ["kill"], "x": 6, "y": 5, "matrix": matrix}) + matrix.add_agent_to_simulation(zombie) + matrix.llm_action(real_agent, matrix.unix_time) + if real_agent.x == 4: + successful_outcomes += 1 + + self.assertTrue(successful_outcomes >= 2) + + def test_matrix_runs_step(self): + matrix = Matrix({"scenario":"configs/empty.json","environment":"configs/largev2.tmj"}) + #run for one step + matrix.steps = 1 + matrix.boot() + matrix.run_singlethread() + self.assertTrue(len(matrix.agents) > 0) + self.assertEqual(matrix.status,"complete") + + def test_perception_range(self): + matrix1 = Matrix({"scenario":"configs/bus_stop.json","environment":"configs/largev2.tmj"}) + matrix1.boot() + self.assertEqual(matrix1.perception_range, 50) # perception_range = 50 in bus_stop.json + + matrix2 = Matrix({"scenario":"configs/empty.json","environment":"configs/largev2.tmj"}) + matrix2.perception_range = 155 + matrix2.boot() + self.assertEqual(matrix2.perception_range, 155) def test_memory(self): agent_data = { @@ -85,7 +113,7 @@ def test_eval(self): print(f"Error {e}") timestamp = datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") - agent1.evaluate_progress(timestamp) + agent1.evaluate_progress({'timestamp':timestamp}) def test_askmetaquestions(self): agent1_data = { @@ -255,13 +283,56 @@ def test_embeddings(self): print(f"Relevance Score: {Memory.calculateRelevanceScore(mem.embedding, context_embedding)}") self.assertTrue(len(sorted_memory) > 0) - def test_information_spreading(self): + + def test_information_dissemination(self): + a1_data = { "name": "Viktor", "goal": "wants to throw a party next week" } + a2_data = { "name": "John", "description": "loves art" } + a3_data = { "name": "Paul", "description": "loves eating" } + agent1 = Agent(a1_data) + agent2 = Agent(a2_data) + agent3 = Agent(a3_data) + unix_time = 1704067200 + timestamp = datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") + for i in range(2): + timestamp = datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") + response = agent1.talk({ "target": agent2.name, "other_agents": [agent2], "timestamp": timestamp }) + msg = f"{agent1} said to {agent2}: {response}" + print(msg) + response = agent2.talk({ "target": agent1.name, "other_agents": [agent1], "timestamp": timestamp }) + msg = f"{agent2} said to {agent1}: {response}" + print(msg) + unix_time = unix_time + 10 + agent2.summarize_conversation(timestamp) + print("*"*20) + for i in range(2): + timestamp = datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") + response = agent2.talk({ "target": agent3.name, "other_agents": [agent3], "timestamp": timestamp }) + msg = f"{agent2} said to {agent3}: {response}" + print(msg) + response = agent3.talk({ "target": agent2.name, "other_agents": [agent2], "timestamp": timestamp }) + msg = f"{agent3} said to {agent2}: {response}" + print(msg) + unix_time = unix_time + 10 + agent3.summarize_conversation(timestamp) + print(f"{agent2} memories") + for mem in agent2.memory: + print(mem) + print(f"{agent3} memories") + for mem in agent3.memory: + print(mem) + + def long_range_test_information_dissemination(self): a1_data = { "name": "Viktor", "goal": "wants to throw a party next week" } a2_data = { "name": "John", "description": "loves art" } a3_data = { "name": "Paul", "description": "loves eating" } agent1 = Agent(a1_data) agent2 = Agent(a2_data) agent3 = Agent(a3_data) + agents = [agent1, agent2, agent3] + for i, agent in enumerate(agents): + for other_agent in agents[i + 1:]: + agent.connections.append(str(other_agent)) + other_agent.connections.append(str(agent)) unix_time = 1704067200 timestamp = datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") for i in range(2): @@ -274,6 +345,11 @@ def test_information_spreading(self): print(msg) unix_time = unix_time + 10 agent2.summarize_conversation(timestamp) + interactions = ["saw a dog" ,"took a nap","chatted with a stranger", "ate lunch", "saw a bird", "saw a friend","saw a stranger", "saw a zombie"] + for i in range(50): + timestamp = datetime.fromtimestamp(unix_time+i).strftime("%Y-%m-%d %H:%M:%S") + interaction = random.choice(interactions) + agent2.addMemory("observation", interaction, timestamp, random.randint(0, 2)) print("*"*20) for i in range(2): timestamp = datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") @@ -572,7 +648,7 @@ def test_agent_decision_zombie(self): agent.destination_cache = [(36, 88), (36, 89), (36, 90)] zombie.destination_cache = [(36, 93), (36, 93), (36, 93)] - agent.addMemory("reflect", f"{matrix.unix_to_strftime(matrix.unix_time)} - {agent} wants to check if the Park is a safe place to go to or no.", matrix.unix_to_strftime(matrix.unix_time), 9) + agent.addMemory("reflect", f"{unix_to_strftime(matrix.unix_time)} - {agent} wants to check if the Park is a safe place to go to or no.", unix_to_strftime(matrix.unix_time), 9) matrix.agents.append(agent) matrix.agents.append(zombie) diff --git a/utils/db_insert.py b/utils/db_insert.py index 7cf4b1f..c0dbba6 100644 --- a/utils/db_insert.py +++ b/utils/db_insert.py @@ -6,10 +6,19 @@ from dotenv import load_dotenv from sshtunnel import SSHTunnelForwarder import redis +import argparse # Load environment variables from .env file load_dotenv() +parser = argparse.ArgumentParser(description='Process and insert data.') +parser.add_argument('jsonl_file_path', type=str, help='Path to the JSONL file') +parser.add_argument('--rows', type=int, help='Number of rows to process', default=None) +parser.add_argument('--id', type=str, help='Override sim_id', default=None) +parser.add_argument('--full', action='store_true', help='Process minimal data only') +args = parser.parse_args() + + # Database settings db_settings = { "database_host": os.environ.get("DB_HOST"), @@ -28,17 +37,35 @@ "ssh_private_key": os.environ.get("SSH_PRIVATE_KEY") } redis_url = os.environ.get("REDIS_URL") +override_sim_id = None +printed = False +inserted = 0 + +def process_and_insert_data(cursor,redis_client, args): + if redis_client: + data_fields = ["sim_id","embedding"] + else: + data_fields = ["step", "substep", "step_type","sim_id","embedding"] -def process_and_insert_data(cursor,redis_client, jsonl_file_path,rows_to_process): - with jsonlines.open(jsonl_file_path, "r") as jsonl_file: + global printed,inserted + with jsonlines.open(args.jsonl_file_path, "r") as jsonl_file: for i, row in enumerate(jsonl_file): - if rows_to_process is None or i < rows_to_process: + if args.rows is None or i < args.rows: obj = json.loads(row) step = obj.get("step") substep = obj.get("substep") step_type = obj.get("step_type") sim_id = obj.get("sim_id") - data = {k: v for k, v in obj.items() if k not in ["step", "substep", "step_type","sim_id","embedding"]} + if args.id: + sim_id = args.id + if printed == False: + print(f"SIM ID: {sim_id}") + if not args.full: + print("inserting minimum data for simulation display") + printed = True + if not args.full and step_type not in ['talk', 'agent_set', 'move', 'matrix_set', 'agent_init']: + continue + data = {k: v for k, v in obj.items() if k not in data_fields} try: if cursor: @@ -49,6 +76,7 @@ def process_and_insert_data(cursor,redis_client, jsonl_file_path,rows_to_process conn.commit() elif redis_client: redis_client.rpush(os.getenv('REDIS_QUEUE', sim_id), json.dumps(data)) + inserted += 1 if i % 500 == 0: print(i) @@ -56,15 +84,14 @@ def process_and_insert_data(cursor,redis_client, jsonl_file_path,rows_to_process if cursor: conn.rollback() print(f"Error: {e}") - print(i) + print(f"cursor: {i}") + print(f"actual inserted: {inserted}") -jsonl_file_path = sys.argv[1] # Assumes the first command-line argument is the JSONL file path -rows_to_process = int(sys.argv[2]) if len(sys.argv) > 2 else None # Assumes the second argument is the number of rows to process cursor = None if redis_url: print("INSERTING INTO REDIS") redis_client = redis.StrictRedis.from_url(redis_url, decode_responses=True) - process_and_insert_data(None,redis_client, jsonl_file_path,rows_to_process) + process_and_insert_data(None,redis_client,args) elif ssh_host: print("INSERTING INTO DB VIA TUNNEL") @@ -83,7 +110,7 @@ def process_and_insert_data(cursor,redis_client, jsonl_file_path,rows_to_process database=db_settings["database_name"], ) cursor = conn.cursor() - process_and_insert_data(cursor,None, jsonl_file_path,rows_to_process) + process_and_insert_data(cursor,None, args) else: print("INSERTING INTO DB") conn = psycopg2.connect( @@ -95,7 +122,7 @@ def process_and_insert_data(cursor,redis_client, jsonl_file_path,rows_to_process ) cursor = conn.cursor() - process_and_insert_data(cursor, None,jsonl_file_path,rows_to_process) + process_and_insert_data(cursor, None,args) if cursor: diff --git a/utils/recorder.py b/utils/recorder.py new file mode 100644 index 0000000..d2d5995 --- /dev/null +++ b/utils/recorder.py @@ -0,0 +1,75 @@ +import os +import sys +import time +from dotenv import load_dotenv +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from obswebsocket import obsws, requests + + +load_dotenv() + +# OBS Studio Websocket Settings +obs_settings = { + "obs_host": os.environ.get("OBS_HOST","127.0.0.1"), + "obs_port": int(os.environ.get("OBS_PORT",4455)), + "obs_password": os.environ.get("OBS_PASSWORD"), + "record_path": os.environ.get("RECORD_PATH","movies"), +} + +# This is NOT working yet +# Start recording based on a javacript event. +def start_recording_if_playing(): + is_playing = driver.execute_script("return isPlaying;") + if is_playing: + start_recording = requests.StartRecord() + ws.call(start_recording) + print("Recording started") + +# Set up Chrome options to start in full-screen mode +chrome_options = Options() +chrome_options.add_argument('--kiosk') +chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']); + + +# Create a WebDriver instance (make sure you have the ChromeDriver executable in your PATH) +driver = webdriver.Chrome(options=chrome_options) + +url_path = sys.argv[1] if len(sys.argv) > 1 else "https://replicantlife.com/" +driver.get(url_path) # Open a website + +# Allow some time for the page to load +time.sleep(3) + +# Connect to OBS Studio via WebSocket +ws = obsws(obs_settings["obs_host"], obs_settings["obs_port"], obs_settings["obs_password"]) +ws.connect() + +# Set the recording path in OBS Studio +recording_path = obs_settings["record_path"] +set_recording_path = requests.SetRecordDirectory(recording_path=recording_path) +ws.call(set_recording_path) + +# Check current recording status +current_recording_status = ws.call(requests.GetRecordStatus()) +print("Current recording status:", current_recording_status) + +# Start recording in OBS Studio +start_recording = requests.StartRecord() +ws.call(start_recording) + +# The function is NOT working yet. +# start_recording_if_playing() + +# Continue recording for 20s (adjust as needed) +time.sleep(20) + +# Stop recording in OBS Studio +stop_recording = requests.StopRecord() +ws.call(stop_recording) + +# Close the browser when done +driver.quit() + +# Disconnect from OBS Studio +ws.disconnect() diff --git a/utils/timeline_info.py b/utils/timeline_info.py index f0ca8bf..7ca2ca4 100644 --- a/utils/timeline_info.py +++ b/utils/timeline_info.py @@ -14,6 +14,10 @@ world = "" last_death_at_step = 0 last_death_at_row = 0 +if "--convo" in sys.argv: + convos = True +else: + convos = False with jsonlines.open(path, "r") as jsonl_file: for i, row in enumerate(jsonl_file): obj = json.loads(row) @@ -21,6 +25,11 @@ step_type = obj.get("step_type") step_types[step_type] = step_types.get(step_type, 0) + 1 sim_id = obj.get("sim_id") + if convos and step_type == "talk": + a = agents[obj["agent_id"]]['name'] + b = agents[obj["to_id"]]['name'] + m = obj["content"] + print(f"{a} said to {b}: {m}") if step_type == "agent_init": agents[obj["agent_id"]] = {} agents[obj["agent_id"]]["name"] = obj["name"] @@ -36,20 +45,23 @@ scenario = obj["data"]["scenario"] world = obj["data"]["environment"] + + total_steps +=1 -human_agents = dict(filter(lambda item: item[1]["kind"] == "human", agents.items())) -print(f"matrix info sim_id: {sim_id}") -print(f"matrix info scenario: {scenario}") -print(f"matrix info world: {world}") -print(f"total individual steps {total_steps}") -print(f"simulation steps {max_step}") -print(f"step types {step_types}") -print(f"average substeps per step {total_steps/max_step}") -print(f"human agent count: {len(human_agents)}") -print(f"deaths: {deaths}") -print(f"last_death_at_step {last_death_at_step}") -print(f"last_death_at_row {last_death_at_row}") -#print(f"human agents {human_agents}") -print(f"total agent count: {len(agents)}") -#print(f"agent actions {agent_actions}") -print(f"completed: {completed}") +if not convos: + human_agents = dict(filter(lambda item: item[1]["kind"] == "human", agents.items())) + print(f"matrix info sim_id: {sim_id}") + print(f"matrix info scenario: {scenario}") + print(f"matrix info world: {world}") + print(f"total individual steps {total_steps}") + print(f"simulation steps {max_step}") + print(f"step types {step_types}") + print(f"average substeps per step {total_steps/max_step}") + print(f"human agent count: {len(human_agents)}") + print(f"deaths: {deaths}") + print(f"last_death_at_step {last_death_at_step}") + print(f"last_death_at_row {last_death_at_row}") + #print(f"human agents {human_agents}") + print(f"total agent count: {len(agents)}") + #print(f"agent actions {agent_actions}") + print(f"completed: {completed}") diff --git a/utils/utils.py b/utils/utils.py index 1ce2351..b79898a 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -3,6 +3,7 @@ import requests import time import os +import re import redis import difflib import random @@ -15,6 +16,7 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker, relationship, column_property import requests +from groq import Groq import nltk nltk.download('brown') @@ -140,6 +142,13 @@ def generate(self, prompt, fallback="Llm Error"): except Exception as e: print(e) msg = fallback + elif re.findall(r'groq',self.model): + #groq,groq_model = self.model.split("_") + client = Groq( api_key=os.environ.get("GROQ_API_KEY"),) + chat_completion = client.chat.completions.create( + messages=[ { "role": "user", "content": prompt, } ], + model="mixtral-8x7b-32768",) + msg = chat_completion.choices[0].message.content else: data = { "model": self.model, @@ -180,7 +189,9 @@ def generate(self, prompt, fallback="Llm Error"): pd(f"current url {current_url}") pd(f"INPUT:\n {prompt}") pd(f"OUTPUT:\n {msg}") - pd(f"runtime: {end_time - start_time}") + print(f"LLM CALL: {prompt[:30]}") + print(f"runtime: {end_time - start_time}") + return msg def embeddings(self, prompt, fallback=[0.5, 0.5, 0.5]): diff --git a/web/.env.development b/web/.env.development index 146797f..3e43d68 100644 --- a/web/.env.development +++ b/web/.env.development @@ -1,3 +1,4 @@ REDIS_URL=redis://localhost:6379 NEXT_PUBLIC_API_URL=http://localhost:3000/api/simulations NEXT_PUBLIC_MOUNTED=false +NEXT_PUBLIC_ASSET_DOMAIN=http://localhost:3004/ diff --git a/web/next.config.mjs b/web/next.config.mjs index 8f767b0..04af33a 100644 --- a/web/next.config.mjs +++ b/web/next.config.mjs @@ -9,12 +9,14 @@ if (process.env.NODE_ENV === 'production') { assetPrefix: '/replay', env: { NEXT_PUBLIC_BASE_PATH: '/replay', + NEXT_PUBLIC_CONTENT_DIRECTORY: '' }, }; } else { nextConfig = { env: { - NEXT_PUBLIC_BASE_PATH: '' + NEXT_PUBLIC_BASE_PATH: '', + NEXT_PUBLIC_CONTENT_DIRECTORY: '' } }; } diff --git a/web/public/images/icons/dead.png b/web/public/images/icons/dead.png new file mode 100644 index 0000000..0b4f16e Binary files /dev/null and b/web/public/images/icons/dead.png differ diff --git a/web/public/images/maps/Southpark.jpg b/web/public/images/maps/Southpark.jpg new file mode 100644 index 0000000..ca6979f Binary files /dev/null and b/web/public/images/maps/Southpark.jpg differ diff --git a/web/src/app/page.tsx b/web/src/app/page.tsx index 71440eb..7d6960d 100644 --- a/web/src/app/page.tsx +++ b/web/src/app/page.tsx @@ -5,18 +5,33 @@ import RenderLevel from '@/components/RenderLevel'; export default function Page() { const [simId, setSimId] = useState(null); + const [map, setMapName] = useState(null); + const [img, setImgName] = useState(null); + const [hidePanel, setHidePanel] = useState(false); useEffect(() => { const params = new URLSearchParams(window.location.search); const sim_id = params.get('sim_id'); + const map = params.get('map'); + const img = params.get('img'); + const hidePanel = params.get('hide'); if (sim_id) { setSimId(sim_id); } + if (map) { + setMapName(map); + } + if (img) { + setImgName(img); + } + if (hidePanel === '1') { + setHidePanel(true); + } }, []); - + if (!simId) { return null; } - return ; + return } diff --git a/web/src/components/Agent.module.css b/web/src/components/Agent.module.css index af800ab..d0094e0 100644 --- a/web/src/components/Agent.module.css +++ b/web/src/components/Agent.module.css @@ -16,10 +16,28 @@ top: -12px; } +.stageLeftChatIcon { + position: absolute; + left: -40px; + top: -60px; +} + +.stageThoughtBubbleIcon { + position: absolute; + right: 40px; + top: -80px; +} + +.stageDeadIcon { + position: absolute; + right: 40px; + top: -80px; +} + .agent { position: relative; } - + .hidden { visibility: hidden; } \ No newline at end of file diff --git a/web/src/components/AgentSprite.tsx b/web/src/components/AgentSprite.tsx index 0fdc21f..4fd9ab7 100644 --- a/web/src/components/AgentSprite.tsx +++ b/web/src/components/AgentSprite.tsx @@ -1,50 +1,93 @@ import React from "react"; import styles from './Agent.module.css'; -const AgentSprite: React.FC<{ agentName: string, isTalking: boolean, isThinking: boolean, status: string }> = ({ agentName, isTalking, isThinking, status }) => { +const AgentSprite: React.FC<{ agentName: string, isTalking: boolean, isThinking: boolean, status: string, map: string }> = ({ agentName, isTalking, isThinking, status, map }) => { let agentImage = `${process.env.NEXT_PUBLIC_BASE_PATH}/images/characters/${agentName}.png`; console.log("AgentSpriteeee status = ", status); + // Check if agentName matches "Zombie" using regex if (/Zombie/.test(agentName)) { agentImage = `${process.env.NEXT_PUBLIC_BASE_PATH}/images/characters/Zombie_1.png`; } - return ( -
-
- Talking - {isThinking && ( + + if (map == "stage") { + return ( +
+
Thinking - )} - {status == "dead" && ( + {isThinking && ( + Thinking + )} + {status == "dead" && ( + Dead + )} +
+ {agentName} +
+ ); + } + else { + return ( +
+
Dead - )} + {isThinking && ( + Thinking + )} + {status == "dead" && ( + Dead + )} +
+ {agentName}
- {agentName} -
- ); + ); + } }; export default AgentSprite; diff --git a/web/src/components/RenderLevel.module.css b/web/src/components/RenderLevel.module.css index db495d3..6e0070b 100644 --- a/web/src/components/RenderLevel.module.css +++ b/web/src/components/RenderLevel.module.css @@ -50,3 +50,29 @@ .sidebarVisible { width: 200px; /*calc(var(--game-viewport-width) * 0.30);*/ } + +.stageImg { + width: 100vw; + height: 100vh; + object-fit: "cover"; +} + +.agentContainer { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + flex-wrap: nowrap; + /* Prevent wrapping */ + justify-content: center; + align-items: flex-end; + padding: 150px; +} + +.agentItem { + margin-right: 20px; + /* Adjust spacing between agents */ + position: relative; +} diff --git a/web/src/components/RenderLevel.tsx b/web/src/components/RenderLevel.tsx index b645c9e..5ac19bb 100644 --- a/web/src/components/RenderLevel.tsx +++ b/web/src/components/RenderLevel.tsx @@ -28,7 +28,8 @@ async function getData(sim_id: string, fromIndex: number) { } // eslint-disable-next-line @typescript-eslint/no-explicit-any -const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { +const RenderLevel: React.FC<{ simId: string, map?: string | null, img?: string | null, hidePanel: boolean }> = ({ simId, map, img, hidePanel }) => { + const [isPlaying, setIsPlaying] = useState(true); const [followAgent, setFollowAgent] = useState(undefined); const [levelState, setLevelState] = useState({ stepId: 0, substepId: 0, agents: [] }); @@ -36,6 +37,7 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { const [initialFetchDone, setInitialFetchDone] = useState(false); const chunkSize = 1000; // Adjust chunk size as needed + const levelRef = useRef(new Level([], (newState: LevelState) => { setLevelState(newState); })); @@ -84,6 +86,30 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { } const renderAgents = () => { + if (map == "stage") { + console.log("STEPID", levelState.stepId); + console.log("AGENT LENGTH", levelState.agents.length); + return levelState.agents.map((agent, index) => { + const x = agent.position.x; + const y = agent.position.y; + + const style: React.CSSProperties = { + position: 'relative', + cursor: 'pointer', + top: x, + left: y, + }; + + return ( +
+ +
+ ); + }); + + } return levelState.agents.map((agent, index) => { const x = agent.position.x * CELL_SIZE; const y = agent.position.y * CELL_SIZE; @@ -100,19 +126,49 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { style={style} className={styles.placement} onClick={() => setFollowAgent(agent)}> - +
); }); }; + if (map == "stage") { + return ( +
+ Stage Map +
+ <> + {renderAgents()} + +
+ +
+ ); + } + return (
- Default Map + Default Map <> {renderAgents()} @@ -123,10 +179,12 @@ const RenderLevel: React.FC<{ simId: string }> = ({ simId }) => { setIsPlaying={setIsPlaying} stepId={levelState.stepId} substepId={levelState.substepId} - level={levelRef.current} /> + level={levelRef.current} + hidePanel={hidePanel} + simId={simId} />
); }; -export default RenderLevel; \ No newline at end of file +export default RenderLevel; diff --git a/web/src/components/Sidebar.module.css b/web/src/components/Sidebar.module.css index 93f0d77..c542c04 100644 --- a/web/src/components/Sidebar.module.css +++ b/web/src/components/Sidebar.module.css @@ -75,4 +75,10 @@ .agentInfo { text-align: justify; +} + +.stepAndAudio { + display: flex; + flex-direction: row; + gap: 12px; } \ No newline at end of file diff --git a/web/src/components/Sidebar.tsx b/web/src/components/Sidebar.tsx index 718117f..eb07fac 100644 --- a/web/src/components/Sidebar.tsx +++ b/web/src/components/Sidebar.tsx @@ -15,7 +15,9 @@ interface SidebarProps { setIsPlaying: React.Dispatch>; stepId: number; substepId: number; + hidePanel: boolean; level: Level; + simId: string; } const Sidebar: React.FC = ( @@ -26,11 +28,23 @@ const Sidebar: React.FC = ( setIsPlaying, stepId, substepId, - level + level, + simId, + hidePanel }) => { - const [showThoughts, setShowThoughts] = useState(true); - + const [showThoughts, setShowThoughts] = useState(true); + const [isPlayAudio, setIsPlayAudio] = useState(true); + const [audioQueue, setAudioQueue] = useState[]>([]); + const [audioPlaying, setAudioPlaying] = useState(false); + const browserLanguage = navigator.language; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const minimal_audio_delay = 500; // delay in between playing audio clips + //hidePanel = false; + //WTF pass this + + useEffect(() => { let interval: NodeJS.Timeout; @@ -48,6 +62,88 @@ const Sidebar: React.FC = ( }; }, [isPlaying]); + const fetchAudioData = async (sim_id: string, step_id: number, substep_id: number, agent_name: string, lang: string, content: string): Promise => { + try { + const res = await fetch( + `${process.env.NEXT_PUBLIC_ASSET_DOMAIN}/audio?mid=${sim_id}&step=${step_id}&substep=${substep_id}&agent=${agent_name}&lang=${lang}&c=${btoa(content)}`, + { mode: 'cors' } + ); + if (!res.ok) { + throw new Error('Failed to fetch data'); + } + + const audioBlob = await res.blob(); + const audioUrl = URL.createObjectURL(audioBlob); + + // Preload the audio file + const audio = new Audio(audioUrl); + audio.preload = 'auto'; + audio.load(); + + return audioUrl; + } catch (error) { + console.error('Error fetching audio:', error); + return ""; + } +}; + + + const addToAudioQueue = (audioClipUrl: Promise) => { + setAudioQueue((oldQueue) => [...oldQueue, audioClipUrl]); + }; + + const playAudio = async (audioClipUrl: Promise) => { + console.log("BEFORE LENGTH", audioQueue.length) + setAudioPlaying(true); + const audio = new Audio(await audioClipUrl); + audio.onended = () => { + setAudioQueue((oldQueue) => oldQueue.slice(1)); + console.log("AFTER LENGTH", audioQueue.length) + setAudioPlaying(false); + }; + audio.play(); + }; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const stopAudio = (audio: HTMLAudioElement) => { + if (audio) { + audio.pause(); + audio.currentTime = 0; + } + }; + + useEffect(() => { + if (isPlaying && agentPlacement && !audioPlaying && audioQueue.length > 0) { + playAudio(audioQueue[0]); + } + }, [agentPlacement, isPlaying, audioQueue, audioPlaying]); + + useEffect(() => { + if (agentPlacement && isPlayAudio) { + const steps = agentPlacement.steps.filter( + step => step.stepId >= stepId + ); + steps.forEach(step => { + if (step instanceof TalkStep) { + const talkStep = step as TalkStep; + addToAudioQueue(fetchAudioData(simId, talkStep.stepId, talkStep.substepId, talkStep.fromAgentName, browserLanguage,talkStep.message)); + + return; + } + if (showThoughts && step instanceof ThoughtStep) { + const thoughtStep = step as ThoughtStep; + addToAudioQueue(fetchAudioData(simId, thoughtStep.stepId, thoughtStep.substepId, thoughtStep.agentId, browserLanguage,talkStep.content)); + return; + } + }); + } + + if (!agentPlacement || !isPlayAudio) { + setAudioQueue([]); + } + + }, [agentPlacement, showThoughts, isPlayAudio, stepId, substepId]); + const handleRewind = (): void => { setIsPlaying(false); setFollowAgent(undefined); @@ -86,6 +182,9 @@ const Sidebar: React.FC = ( const renderTimeline = () => { if(!agentPlacement) return null; + if (hidePanel) { + return null; + } const steps = agentPlacement.steps.toReversed(); @@ -108,10 +207,19 @@ const Sidebar: React.FC = ( }; const renderControls = () => { + if (hidePanel) { + return null; + } return(
-
+
Step: {stepId} + {process.env.NEXT_PUBLIC_ALLOW_AUDIO === "true" &&
+ +
}
{substepId}
@@ -125,6 +233,9 @@ const Sidebar: React.FC = (
) }; + if (hidePanel) { + return null; + } return ( // JSX for the Sidebar component goes here @@ -158,4 +269,4 @@ const Sidebar: React.FC = ( ); }; -export default Sidebar; \ No newline at end of file +export default Sidebar;