elias_total_loop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "import brikasutils as bu\n",
    "importlib.reload(bu)\n",
    "import shared_utils as utils\n",
    "from shared_utils import systemMsg, userMsg, assistantMsg\n",
    "importlib.reload(utils)\n",
    "import survey\n",
    "importlib.reload(survey)\n",
    "import persona\n",
    "importlib.reload(persona)\n",
    "import ollama\n",
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "from typing import List\n",
    "from openai import OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 1916 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-09-13 to 2024-03-06\n",
      "Messages saved to self.chats['airidas']\n",
      "Read 618 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-09-10 to 2024-03-03\n",
      "Messages saved to self.chats['christian']\n",
      "Read 297 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2018-07-25 to 2024-01-01\n",
      "Messages saved to self.chats['nikolay']\n",
      "Read 144 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-03-28 to 2021-12-30\n",
      "Messages saved to self.chats['mathis']\n",
      "Read 104 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-08-25 to 2024-03-05\n",
      "Messages saved to self.chats['jacob']\n",
      "Read 159 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-10-12 to 2023-04-30\n",
      "Messages saved to self.chats['chris']\n",
      "Read 161 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-03-28 to 2021-06-06\n",
      "Messages saved to self.chats['aziz']\n",
      "Read 350 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-03-26 to 2022-04-11\n",
      "Messages saved to self.chats['daniela']\n",
      "Read 105 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-03-28 to 2022-07-10\n",
      "Messages saved to self.chats['mihi']\n",
      "Read 117 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-05-17 to 2022-04-11\n",
      "Messages saved to self.chats['viktoria']\n",
      "Read 172 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2019-08-11 to 2023-12-21\n",
      "Messages saved to self.chats['diba']\n",
      "Read 154 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2021-12-02 to 2023-04-29\n",
      "Messages saved to self.chats['filip']\n",
      "Read 659 messages from 1 files. Failed to read 0 messages.\n",
      "Messages ranged from 2023-03-12 to 2024-02-16\n",
      "Messages saved to self.chats['rebecca']\n",
      "Filtering\n",
      "link-filter: 133\n",
      "react-filter: 0\n",
      "danish-react-filter: 66\n",
      "cookie-data-filter: 12\n",
      "Selected chat rebecca for 11000 (634 messages)\n",
      "Selected chat airidas for 26025 (1671 messages)\n",
      "Selected chat christian for 5251 (576 messages)\n",
      "Selected chat nikolay for 3128 (266 messages)\n",
      "Selected chat mathis for 1220 (127 messages)\n",
      "Selected chat daniela for 2716 (282 messages)\n",
      "Selected chat diba for 1589 (142 messages)\n",
      "Selected chat aziz for 934 (117 messages)\n",
      "Selected chat jacob for 1124 (93 messages)\n",
      "Selected chat chris for 1534 (151 messages)\n",
      "Selected chat filip for 1305 (146 messages)\n",
      "Selected chat mihi for 1008 (96 messages)\n",
      "Selected chat viktoria for 1033 (98 messages)\n",
      "Combined tokens: 57867\n"
     ]
    }
   ],
   "source": [
    "et = persona.PersonaEncoder()\n",
    "et.parse_fb_messages([\"data/1_raw/1_airidas.json\"], \"airidas\")\n",
    "et.parse_fb_messages([\"data/1_raw/2_christian.json\"], \"christian\")\n",
    "et.parse_fb_messages([\"data/1_raw/1_nikolay.json\"], \"nikolay\")\n",
    "et.parse_fb_messages([\"data/1_raw/2_mathis.json\"], \"mathis\")\n",
    "et.parse_fb_messages([\"data/1_raw/2_jacob.json\"], \"jacob\")\n",
    "et.parse_fb_messages([\"data/1_raw/2_chris.json\"], \"chris\")\n",
    "et.parse_fb_messages([\"data/1_raw/3_aziz.json\"], \"aziz\")\n",
    "et.parse_fb_messages([\"data/1_raw/3_daniela.json\"], \"daniela\")\n",
    "et.parse_fb_messages([\"data/1_raw/3_mihi.json\"], \"mihi\")\n",
    "et.parse_fb_messages([\"data/1_raw/3_viktoria.json\"], \"viktoria\")\n",
    "et.parse_fb_messages([\"data/1_raw/4_diba.json\"], \"diba\")\n",
    "et.parse_fb_messages([\"data/1_raw/6_filip.json\"], \"filip\")\n",
    "et.parse_wa_messages([\"data/1_raw/messages_1000.json\"], \"rebecca\")\n",
    "et.filter_chats_empty()\n",
    "et.filter_chats_regex(utils.BLACKLIST_CHAT_REGEX_FILTERS)\n",
    "for nameid, chat in et.chats.items():\n",
    "    for msg in chat:  \n",
    "        msg.sender = \"Persona\" if msg.sender == \"Elias Salvador Smidt Torjani\"  else \"Friend\"\n",
    "et.select_chat_full(\"rebecca\")\n",
    "et.select_chat_full(\"airidas\")\n",
    "et.select_chat_full(\"christian\")\n",
    "et.select_chat_full(\"nikolay\")\n",
    "et.select_chat_full(\"mathis\") \n",
    "et.select_chat_full(\"daniela\")\n",
    "et.select_chat_full(\"diba\")\n",
    "et.select_chat_full(\"aziz\")\n",
    "et.select_chat_full(\"jacob\")  \n",
    "et.select_chat_full(\"chris\")\n",
    "et.select_chat_full(\"filip\")\n",
    "et.select_chat_full(\"mihi\")\n",
    "et.select_chat_full(\"viktoria\")\n",
    "token_counts = et.count_all_selected_chat_tokens() # token_counts used later for statistics\n",
    "print(f\"Combined tokens: {sum(token_counts.values())}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Boss POST-DANIEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chunk 53/53"
     ]
    }
   ],
   "source": [
    "CHUNK_SIZES = [75]\n",
    "chunk_size = CHUNK_SIZES[0]\n",
    "EMBED_MODEL = \"nomic-embed-text\"\n",
    "OVERLAP_SIZES = [3]\n",
    "overlap_size = OVERLAP_SIZES[0]\n",
    "\n",
    "chunks = []\n",
    "chunk_token_counts = []\n",
    "for chat in et.selectedChats.values():\n",
    "    messages = list(chat)  # Convert chat iterator to list for easier slicing\n",
    "    num_messages = len(messages)\n",
    "    for i in range(0, num_messages - chunk_size + 1, chunk_size - overlap_size):\n",
    "        chunk = messages[i:i + chunk_size]  # Extract chunk of messages\n",
    "        chunk_text = \"\\\\n\".join(str(msg) for msg in chunk)  # Concatenate msgs into a single string\n",
    "        chunks.append(chunk_text)  # Append chunk to list of chunks\n",
    "        chunk_token_counts.append(utils.count_tokens(chunk_text))  # Append token count of the chunk\n",
    "avg_chunk_token_count = sum(chunk_token_counts) / len(chunk_token_counts)\n",
    "embeddings = []\n",
    "progress, chunks_len = 0, len(chunks)\n",
    "\n",
    "for chunk_text in chunks:\n",
    "    progress += 1\n",
    "    print(f\"\\rChunk {progress}/{chunks_len}\", end=\"\")\n",
    "    embedding = ollama.embeddings(model=EMBED_MODEL, prompt=chunk_text)[\"embedding\"]\n",
    "    embeddings.append(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using default Kano Survey CSV file: surveys/survey_kano-model.csv\n",
      "Using default Personality Survey CSV file: surveys/survey_personality-test.csv\n",
      "Tokens in average chunk group: 42.9\n",
      "Tokens in average chunk group: 173.1904761904762\n",
      "Tokens in average chunk group: 331.3636363636364\n",
      "Tokens in average chunk group: 37.30434782608695\n",
      "Tokens in average chunk group: 154.0\n",
      "Tokens in average chunk group: 288.76\n"
     ]
    }
   ],
   "source": [
    "PROMPT_METHOD =\"IMPERSONATE\"\n",
    "SUBJECT = \"eli\"\n",
    "RETRIAVAL_METHODS = [\"static\"]#, \"hybrid\", \"dynamic\", \n",
    "NUM_RUNS = 3\n",
    "MODEL = \"llama3-70b\"\n",
    "# MODEL = \"llama3-8b\"\n",
    "# MODEL = \"mixtral-8x22b\"\n",
    "# max_tokens = [7500]\n",
    "max_tokens = [0, 4000, 7500]\n",
    "\n",
    "survs = [survey.KanoSurvey(), survey.PersonalitySurvey()]\n",
    "for surv in survs:\n",
    "    if isinstance(surv, survey.KanoSurvey):\n",
    "        DYNAMIC_RETRIEVAL_PROMPTS = list(surv.questions)\n",
    "        PROMPT_COUNT = 40\n",
    "        SURVEY_TYPE = \"KanoSurvey\",\n",
    "        WHICH_SURVEY = \"kano\"\n",
    "        RETRIEVAL_PROMPT = \"video game features\"\n",
    "        SURVEY = \"video game preferences\"\n",
    "        METHOD = \"a Kano survey\"\n",
    "    elif isinstance(surv, survey.PersonalitySurvey):\n",
    "        DYNAMIC_RETRIEVAL_PROMPTS = list(surv.questions)\n",
    "        PROMPT_COUNT = 50\n",
    "        SURVEY_TYPE = \"PersonalitySurvey\",\n",
    "        WHICH_SURVEY = \"pers\"\n",
    "        RETRIEVAL_PROMPT = \"openess conciousness extrovert aggreableness neuroticism\"\n",
    "        SURVEY = \"personality traits\"\n",
    "        METHOD = \"an OCEAN test\"\n",
    "\n",
    "    for max_token in max_tokens:\n",
    "        for RETRIAVAL_METHOD in RETRIAVAL_METHODS:\n",
    "            if RETRIAVAL_METHOD == \"dynamic\":\n",
    "                dynamic_retrieval_prompts = list(surv.questions)\n",
    "                dynamic_chunks_most_similar: List[List[str]] = [] \n",
    "                progress = 0\n",
    "                lenn = len(dynamic_retrieval_prompts)\n",
    "                for prompt in dynamic_retrieval_prompts:\n",
    "                    progress += 1\n",
    "                    print(f\"\\rPrompt {progress}/{lenn}\", end=\"\")\n",
    "                    prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=prompt)[\"embedding\"]\n",
    "                    ## Chunking\n",
    "                    max_chunks_count = int((max_token / avg_chunk_token_count))\n",
    "                    chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)\n",
    "                    chunks_most_similar = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings]\n",
    "                    if max_token == 0:\n",
    "                        chunks_most_similar = chunks_most_similar[:1]\n",
    "                    else:\n",
    "                        # Limit chunks by man token count\n",
    "                        cur_tc = 0 # current token count\n",
    "                        selected_chunks = []\n",
    "                        for chunk in chunks_most_similar:\n",
    "                            tk_in_chunk = utils.count_tokens(chunk)\n",
    "                            if cur_tc + tk_in_chunk >= max_token:\n",
    "                                break\n",
    "                            cur_tc += tk_in_chunk\n",
    "                            selected_chunks.append(chunk)\n",
    "                        chunks_most_similar = selected_chunks\n",
    "                    tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in chunks_most_similar)\n",
    "                    dynamic_chunks_most_similar.append(chunks_most_similar)\n",
    "                print(end=\"\\n\")\n",
    "                tokens_in_chunks = 0\n",
    "                for chunks_most_similar in dynamic_chunks_most_similar:\n",
    "                    for chunk in chunks_most_similar:\n",
    "                        tokens_in_chunks += utils.count_tokens(chunk)\n",
    "                # del chunks_most_similar_embeddings  # free memory\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks/len(dynamic_chunks_most_similar)}\")\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "for question, chunks_most_similar in zip(surv.questions, dynamic_chunks_most_similar):\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famous people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),     \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "\n",
    "\n",
    "            elif RETRIAVAL_METHOD == \"static\":\n",
    "                prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=RETRIEVAL_PROMPT)[\"embedding\"]\n",
    "                chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)\n",
    "                chunks_most_similar = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings]\n",
    "                ## Chunking\n",
    "                if max_token == 0:\n",
    "                    chunks_most_similar = chunks_most_similar[:1]\n",
    "                else:\n",
    "                    # Limit chunks by man token count\n",
    "                    cur_tc = 0 # current token count\n",
    "                    selected_chunks = []\n",
    "                    for chunk in chunks_most_similar:\n",
    "                        tk_in_chunk = utils.count_tokens(chunk)\n",
    "                        if cur_tc + tk_in_chunk >= max_token:\n",
    "                            break\n",
    "                        cur_tc += tk_in_chunk\n",
    "                        selected_chunks.append(chunk)\n",
    "                    chunks_most_similar = selected_chunks\n",
    "                tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in chunks_most_similar)\n",
    "                static_chunks_most_similar.append(chunks_most_similar)\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks/len(static_chunks_most_similar)}\")\n",
    "                del chunks_most_similar_embeddings  \n",
    "                #Retrieve^\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "for question in surv.questions:\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famous people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),     \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "            \n",
    "            else: print(\"neither static, nor dynamic\")\n",
    "            exec(prompt_template)\n",
    "            SIM_ID = f\"{SUBJECT}-{WHICH_SURVEY}-{RETRIAVAL_METHOD}-{max_token}_{MODEL}_V8\"\n",
    "            bu.quickJSON(final_prompts, f\"data/temp/{SIM_ID}_prompts.json\")\n",
    "            for num_run in range(NUM_RUNS):\n",
    "                instructions = {\n",
    "                    \"prompt_file\": f\"batch/prompts/{SIM_ID}_prompts.json\",\n",
    "                    \"survey_type\": f\"{SURVEY_TYPE[0]}\",\n",
    "                    \"isLocal\": True,\n",
    "                    \"LIMIT\": None\n",
    "                }\n",
    "                settings = {\n",
    "                    \"model\": MODEL,\n",
    "                    \"timeout\": 300}\n",
    "                AUTO_INFO = {\n",
    "                    \"CHUNK_SIZE\": chunk_size,\n",
    "                    \"OVERLAP_SIZE\": overlap_size,\n",
    "                    \"CTX_limit\": max_token,\n",
    "                    \"chunk_count\": len(chunks_most_similar),\n",
    "                    \"EMBED_MODEL\": EMBED_MODEL,\n",
    "                    \"prompt method\": PROMPT_METHOD,\n",
    "                    \"retrieval method\": RETRIAVAL_METHOD,\n",
    "                    \"RETRIEVAL_PROMPT\": RETRIEVAL_PROMPT,\n",
    "                    \"prompt_count\": PROMPT_COUNT,\n",
    "                    \"survey\": WHICH_SURVEY,\n",
    "                    \"SUBJECT\": SUBJECT,\n",
    "                    \"prompt_template\": prompt_template,\n",
    "                    \"CHUNKS_COUNT_IN_CTX\": max_chunks_count,\n",
    "                    **utils.describe_prompts(final_prompts)\n",
    "                }\n",
    "                bu.quickJSON({\"instructions\": instructions, \"settings\": settings, \"info\": AUTO_INFO}, f\"data/temp/runs/{SIM_ID}_{num_run}.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hybrid SHIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_chunks_with_limit(static_chunks_most_similar, dynamic_chunks_most_similar, max_token):\n",
    "  \"\"\"Combines chunks from static and dynamic methods while limiting by max_token.\n",
    "\n",
    "  Args:\n",
    "      static_chunks_most_similar: List of strings representing static retrieval chunks.\n",
    "      dynamic_chunks_most_similar: List of strings representing dynamic retrieval chunks.\n",
    "      max_token: The maximum allowed token count.\n",
    "\n",
    "  Returns:\n",
    "      A list of strings representing the combined and limited chunks.\n",
    "  \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROMPT_METHOD =\"IMPERSONATE\"\n",
    "SUBJECT = \"eli\"\n",
    "RETRIAVAL_METHODS = [\"hybrid\"]#, \"static\", \"dynamic\", \n",
    "NUM_RUNS = 3\n",
    "# MODEL = \"llama3-70b\"\n",
    "MODEL = \"llama3-8b\"\n",
    "# MODEL = \"mixtral-8x22b\"\n",
    "max_tokens = [0, 2000, 5000]\n",
    "# max_tokens = [7500]\n",
    "# max_tokens = [0, 4000, 7500]\n",
    "\n",
    "survs = [survey.KanoSurvey(), survey.PersonalitySurvey()]\n",
    "for surv in survs:\n",
    "    if isinstance(surv, survey.KanoSurvey):\n",
    "        DYNAMIC_RETRIEVAL_PROMPTS = list(surv.questions)\n",
    "        PROMPT_COUNT = 40\n",
    "        SURVEY_TYPE = \"KanoSurvey\",\n",
    "        WHICH_SURVEY = \"kano\"\n",
    "        RETRIEVAL_PROMPT = \"video game features\"\n",
    "        SURVEY = \"video game preferences\"\n",
    "        METHOD = \"a Kano survey\"\n",
    "    elif isinstance(surv, survey.PersonalitySurvey):\n",
    "        DYNAMIC_RETRIEVAL_PROMPTS = list(surv.questions)\n",
    "        PROMPT_COUNT = 50\n",
    "        SURVEY_TYPE = \"PersonalitySurvey\",\n",
    "        WHICH_SURVEY = \"pers\"\n",
    "        RETRIEVAL_PROMPT = \"openess conciousness extrovert aggreableness neuroticism\"\n",
    "        SURVEY = \"personality traits\"\n",
    "        METHOD = \"an OCEAN test\"\n",
    "\n",
    "    for max_token in max_tokens:\n",
    "\n",
    "        for RETRIAVAL_METHOD in RETRIAVAL_METHODS:\n",
    "\n",
    "            if RETRIAVAL_METHOD == \"dynamic\":\n",
    "                dynamic_retrieval_prompts = list(surv.questions)\n",
    "                dynamic_chunks_most_similar: List[List[str]] = [] \n",
    "                progress = 0\n",
    "                lenn = len(dynamic_retrieval_prompts)\n",
    "                for prompt in dynamic_retrieval_prompts:\n",
    "                    progress += 1\n",
    "                    print(f\"\\rPrompt {progress}/{lenn}\", end=\"\")\n",
    "                    prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=prompt)[\"embedding\"]\n",
    "                    ## Chunking\n",
    "                    max_chunks_count = int((max_token / avg_chunk_token_count))\n",
    "                    chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)\n",
    "                    chunks_most_similar = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings]\n",
    "                    if max_token == 0:\n",
    "                        chunks_most_similar = chunks_most_similar[:1]\n",
    "                    else:\n",
    "                        # Limit chunks by man token count\n",
    "                        cur_tc = 0 # current token count\n",
    "                        selected_chunks = []\n",
    "                        for chunk in chunks_most_similar:\n",
    "                            tk_in_chunk = utils.count_tokens(chunk)\n",
    "                            if cur_tc + tk_in_chunk >= max_token:\n",
    "                                break\n",
    "                            cur_tc += tk_in_chunk\n",
    "                            selected_chunks.append(chunk)\n",
    "                        chunks_most_similar = selected_chunks\n",
    "                    tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in chunks_most_similar)\n",
    "                    dynamic_chunks_most_similar.append(chunks_most_similar)\n",
    "                print(end=\"\\n\")\n",
    "                tokens_in_chunks = 0\n",
    "                for chunks_most_similar in dynamic_chunks_most_similar:\n",
    "                    for chunk in chunks_most_similar:\n",
    "                        tokens_in_chunks += utils.count_tokens(chunk)\n",
    "                # del chunks_most_similar_embeddings  # free memory\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks/len(dynamic_chunks_most_similar)}\")\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "for question, chunks_most_similar in zip(surv.questions, dynamic_chunks_most_similar):\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famous people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),     \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "\n",
    "\n",
    "            elif RETRIAVAL_METHOD == \"static\":\n",
    "                prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=RETRIEVAL_PROMPT)[\"embedding\"]\n",
    "                chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)\n",
    "                chunks_most_similar = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings]\n",
    "                ## Chunking\n",
    "                if max_token == 0:\n",
    "                    chunks_most_similar = chunks_most_similar[:1]\n",
    "                else:\n",
    "                    # Limit chunks by man token count\n",
    "                    cur_tc = 0 # current token count\n",
    "                    selected_chunks = []\n",
    "                    for chunk in chunks_most_similar:\n",
    "                        tk_in_chunk = utils.count_tokens(chunk)\n",
    "                        if cur_tc + tk_in_chunk >= max_token:\n",
    "                            break\n",
    "                        cur_tc += tk_in_chunk\n",
    "                        selected_chunks.append(chunk)\n",
    "                    chunks_most_similar = selected_chunks\n",
    "                \n",
    "                tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in chunks_most_similar)\n",
    "                static_chunks_most_similar.append(chunks_most_similar)\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks/len(static_chunks_most_similar)}\")\n",
    "                del chunks_most_similar_embeddings  \n",
    "                #Retrieve^\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "for question in surv.questions:\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famous people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),     \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "\n",
    "\n",
    "            elif RETRIAVAL_METHOD == \"hybrid\":\n",
    "                # Access global static chunks\n",
    "                hybrid_chunks_most_similar: List[List[str]] = []\n",
    "\n",
    "                # Retrieve chunks for RETRIEVAL_PROMPT\n",
    "                prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=RETRIEVAL_PROMPT)[\"embedding\"]\n",
    "                global_static_chunks_most_similar = [chunks[embedding[1]] for embedding in utils.find_most_similar(prompt_embedding, embeddings)]\n",
    "                # retrieval_prompt_chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)\n",
    "                # static_chunks_most_similar = [chunks[embedding[1]] for embedding in retrieval_prompt_chunks_most_similar_embeddings]\n",
    "                for chunk in global_static_chunks_most_similar:\n",
    "                    p.append(chunk)\n",
    "\n",
    "                # Retrieve chunks for prompt\n",
    "                prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=prompt)[\"embedding\"]\n",
    "                prompt_chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)\n",
    "                dynamic_chunks_most_similar = [chunks[embedding[1]] for embedding in prompt_chunks_most_similar_embeddings]\n",
    "\n",
    "                if max_token == 0:\n",
    "                    p.append([static_chunks_most_similar[0], dynamic_chunks_most_similar[0]])  # Edge case: 1 chunk from each\n",
    "                \n",
    "                # Combine chunks while limiting by max_token\n",
    "                combined_chunks = []\n",
    "                cur_tc = 0  # current token count\n",
    "                for chunk in global_static_chunks_most_similar + dynamic_chunks_most_similar:\n",
    "                    tk_in_chunk = utils.count_tokens(chunk)\n",
    "                    if cur_tc + tk_in_chunk >= max_token:\n",
    "                        break\n",
    "                    cur_tc += tk_in_chunk\n",
    "                    combined_chunks.append(chunk)\n",
    "\n",
    "                hybrid_chunks_most_similar.append(combined_chunks)\n",
    "                tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in combined_chunks)\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks/len(hybrid_chunks_most_similar)}\")\n",
    "                #Retrieve^\n",
    "                # for question in surv.questions:\n",
    "                #     p = [question]\n",
    "                # for chunk in chunks_most_similar[i]:\n",
    "                #     p.append(chunk)\n",
    "                #     i += 1\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "p = []\n",
    "for chunk in chunks_most_similar:\n",
    "    p.append(chunk)\n",
    "\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famous people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),     \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "        \n",
    "            else: print(\"neither static, dynamic, nor hybrid\")\n",
    "            \n",
    "            exec(prompt_template)\n",
    "            SIM_ID = f\"{SUBJECT}-{WHICH_SURVEY}-{RETRIAVAL_METHOD}-{max_token}_{MODEL}_V7\"\n",
    "            bu.quickJSON(final_prompts, f\"data/temp/{SIM_ID}_prompts.json\")\n",
    "            for num_run in range(NUM_RUNS):\n",
    "                instructions = {\n",
    "                    \"prompt_file\": f\"batch/prompts/{SIM_ID}_prompts.json\",\n",
    "                    \"survey_type\": f\"{SURVEY_TYPE[0]}\",\n",
    "                    \"isLocal\": True,\n",
    "                    \"LIMIT\": None\n",
    "                }\n",
    "                settings = {\n",
    "                    \"model\": MODEL,\n",
    "                    \"timeout\": 300}\n",
    "                AUTO_INFO = {\n",
    "                    \"CHUNK_SIZE\": chunk_size,\n",
    "                    \"OVERLAP_SIZE\": overlap_size,\n",
    "                    \"CTX_limit\": max_token,\n",
    "                    \"chunk_count\": len(chunks_most_similar),\n",
    "                    \"EMBED_MODEL\": EMBED_MODEL,\n",
    "                    \"prompt method\": PROMPT_METHOD,\n",
    "                    \"retrieval method\": RETRIAVAL_METHOD,\n",
    "                    \"RETRIEVAL_PROMPT\": RETRIEVAL_PROMPT,\n",
    "                    \"prompt_count\": PROMPT_COUNT,\n",
    "                    \"survey\": WHICH_SURVEY,\n",
    "                    \"SUBJECT\": SUBJECT,\n",
    "                    \"prompt_template\": prompt_template,\n",
    "                    \"CHUNKS_COUNT_IN_CTX\": max_chunks_count,\n",
    "                    **utils.describe_prompts(final_prompts)\n",
    "                }\n",
    "                bu.quickJSON({\"instructions\": instructions, \"settings\": settings, \"info\": AUTO_INFO}, f\"data/temp/runs/{SIM_ID}_{num_run}.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SHIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_tokens = 8000\n",
    "cur_tc = 0 # current token count\n",
    "selected_chunks = []\n",
    "for chunk in chunks_most_similar:\n",
    "    tk_in_chunk = utils.count_tokens(chunk)\n",
    "    if cur_tc + tk_in_chunk >= max_tokens:\n",
    "        break\n",
    "    cur_tc += tk_in_chunk\n",
    "    selected_chunks.append(chunk)\n",
    "print(f\"Tokens in chunks: {cur_tc}\")\n",
    "len(selected_chunks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MODEL = \"llama3-70b\"\n",
    "# MODEL = \"llama3-8b\"\n",
    "MODEL = \"mixtral-8x22b\"\n",
    "RETRIAVAL_METHODS = [\"base\"]#, \"hybrid\"]\n",
    "SUBJECT = \"base\"\n",
    "PROMPT_METHOD =\"IMPERSONATE\"\n",
    "NUM_RUNS = 3\n",
    "\n",
    "CHUNK_SIZES = [75]\n",
    "chunk_size = CHUNK_SIZES[0]\n",
    "OVERLAP_SIZES = [3]\n",
    "overlap_size = OVERLAP_SIZES[0]\n",
    "\n",
    "survs = [survey.KanoSurvey(), survey.PersonalitySurvey()]\n",
    "for surv in survs:\n",
    "    if isinstance(surv, survey.KanoSurvey):\n",
    "        DYNAMIC_RETRIEVAL_PROMPTS = list(surv.questions)\n",
    "        PROMPT_COUNT = 40\n",
    "        SURVEY_TYPE = \"KanoSurvey\",\n",
    "        WHICH_SURVEY = \"kano\"\n",
    "        RETRIEVAL_PROMPT = \"video game features\"\n",
    "        SURVEY = \"video game preferences\"\n",
    "        METHOD = \"a Kano survey\"\n",
    "    elif isinstance(surv, survey.PersonalitySurvey):\n",
    "        DYNAMIC_RETRIEVAL_PROMPTS = list(surv.questions)\n",
    "        PROMPT_COUNT = 50\n",
    "        SURVEY_TYPE = \"PersonalitySurvey\",\n",
    "        WHICH_SURVEY = \"pers\"\n",
    "        RETRIEVAL_PROMPT = \"openess conciousness extrovert aggreableness neuroticism\"\n",
    "        SURVEY = \"personality traits\"\n",
    "        METHOD = \"an OCEAN test\"\n",
    "    for RETRIAVAL_METHOD in RETRIAVAL_METHODS:\n",
    "        if RETRIAVAL_METHOD == \"base\":\n",
    "            final_prompts = []\n",
    "            prompt_template = \"\"\"\n",
    "for question in surv.questions:\n",
    "    p = [\n",
    "        systemMsg(\n",
    "            \"You are participating in a survey. You will be presented with a series of questions about your {SURVEY}.\",\n",
    "            f\"You must choose answer to the question below with one of the five options: {', '.join(surv.POSSIBLE_ANSWERS)}. The answer must only contain the chosen option. \"\n",
    "        ),\n",
    "        assistantMsg('Understood. I will answer the question below with one of the given options.'),\n",
    "        userMsg(\n",
    "            question,\n",
    "            \"Your choice: \"\n",
    "        )]\n",
    "    final_prompts.append(p)\n",
    "            \"\"\"\n",
    "        else: print(\"not ???\")\n",
    "        exec(prompt_template)\n",
    "        SIM_ID = f\"{SUBJECT}-{WHICH_SURVEY}-{str(max_chunks_count).zfill(2)}_{MODEL}_V7\"\n",
    "        bu.quickJSON(final_prompts, f\"data/5_monster_prep/{SIM_ID}_prompts.json\")\n",
    "        for num_run in range(NUM_RUNS):\n",
    "            instructions = {\n",
    "                \"prompt_file\": f\"batch/prompts/{SIM_ID}_prompts.json\",\n",
    "                \"survey_type\": f\"{SURVEY_TYPE[0]}\",\n",
    "                \"isLocal\": True,\n",
    "                \"LIMIT\": None\n",
    "            }\n",
    "            settings = {\n",
    "                \"model\": MODEL,\n",
    "                \"timeout\": 300}\n",
    "            AUTO_INFO = {\n",
    "                \"CHUNK_SIZE\": chunk_size,\n",
    "                \"OVERLAP_SIZE\": overlap_size,\n",
    "                \"CTX_limit\": max_token,\n",
    "                \"EMBED_MODEL\": EMBED_MODEL,\n",
    "                \"prompt method\": PROMPT_METHOD,\n",
    "                \"retrieval method\": RETRIAVAL_METHOD,\n",
    "                \"RETRIEVAL_PROMPT\": RETRIEVAL_PROMPT,\n",
    "                \"prompt_count\": PROMPT_COUNT,\n",
    "                \"survey\": WHICH_SURVEY,\n",
    "                \"SUBJECT\": SUBJECT,\n",
    "                \"prompt_template\": prompt_template,\n",
    "                \"CHUNKS_COUNT_IN_CTX\": max_chunks_count,\n",
    "                **utils.describe_prompts(final_prompts)\n",
    "            }\n",
    "            bu.quickJSON({\"instructions\": instructions, \"settings\": settings, \"info\": AUTO_INFO}, f\"data/5_monster_prep/batch/{SIM_ID}_{num_run}.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### V5/6 - 32k models w/ timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHUNK_SIZES = [250]\n",
    "MODELS = [\"mixtral:8x22b-instruct-v0.1-q2_K\"]\n",
    "# OVERLAP_SIZES = [5]\n",
    "overlap_size = 5\n",
    "RETRIAVAL_METHODS = [\"dynamic\", \"hybrid\"]\n",
    "max_tokens = 10000\n",
    "for chunk_size in CHUNK_SIZES:\n",
    "    for MODEL in MODELS:\n",
    "        chunks = []\n",
    "        chunk_token_counts = []\n",
    "        for chat in et.selectedChats.values():\n",
    "            messages = list(chat)  # Convert chat iterator to list for easier slicing\n",
    "            num_messages = len(messages)\n",
    "            for i in range(0, num_messages - chunk_size + 1, chunk_size - overlap_size):\n",
    "                chunk = messages[i:i + chunk_size]  # Extract chunk of messages\n",
    "                chunk_text = \"\\\\n\".join(str(msg) for msg in chunk)  # Concatenate msgs into a single string\n",
    "                chunks.append(chunk_text)  # Append chunk to list of chunks\n",
    "                chunk_token_counts.append(utils.count_tokens(chunk_text))  # Append token count of the chunk\n",
    "        avg_chunk_token_count = sum(chunk_token_counts) / len(chunk_token_counts)\n",
    "        embeddings = []\n",
    "        progress, chunks_len = 0, len(chunks)\n",
    "        for chunk_text in chunks:\n",
    "            progress += 1\n",
    "            print(f\"\\rChunk {progress}/{chunks_len}\", end=\"\")\n",
    "            embedding = ollama.embeddings(model=EMBED_MODEL, prompt=chunk_text)[\"embedding\"]\n",
    "            embeddings.append(embedding)\n",
    "        for RETRIAVAL_METHOD in RETRIAVAL_METHODS:\n",
    "            if RETRIAVAL_METHOD == \"dynamic\":\n",
    "                dynamic_retrieval_prompts = list(surv.questions)\n",
    "                dynamic_chunks_most_similar: List[List[str]] = [] \n",
    "                progress = 0\n",
    "                lenn = len(dynamic_retrieval_prompts)\n",
    "                for prompt in dynamic_retrieval_prompts:\n",
    "                    progress += 1\n",
    "                    print(f\"\\rPrompt {progress}/{lenn}\", end=\"\")\n",
    "                    prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=prompt)[\"embedding\"]\n",
    "                    max_chunks_count = int((max_tokens / avg_chunk_token_count)-1)\n",
    "                    chunks_most_similar_embeddings = utils.find_most_similar(prompt_embedding, embeddings)[:max_chunks_count]\n",
    "                    chunks_most_similar = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings]\n",
    "                    tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in chunks_most_similar)\n",
    "                    dynamic_chunks_most_similar.append(chunks_most_similar)\n",
    "                print(end=\"\\n\")\n",
    "                tokens_in_chunks = 0\n",
    "                for chunks_most_similar in dynamic_chunks_most_similar:\n",
    "                    for chunk in chunks_most_similar:\n",
    "                        tokens_in_chunks += utils.count_tokens(chunk)\n",
    "                del chunks_most_similar_embeddings  # free memory\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks/len(dynamic_chunks_most_similar)}\")\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "for question, chunks_most_similar in zip(surv.questions, dynamic_chunks_most_similar):\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famouns people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n**From most to least related**\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),      \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in an {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "            elif RETRIAVAL_METHOD == \"hybrid\":\n",
    "                prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=RETRIEVAL_PROMPT)[\"embedding\"]\n",
    "                max_chunks_count = int((max_tokens / avg_chunk_token_count))\n",
    "                chunks_most_similar_embeddings_static = utils.find_most_similar(prompt_embedding, embeddings)[:max_chunks_count // 2]\n",
    "                chunks_most_similar_static = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings_static]\n",
    "\n",
    "                dynamic_retrieval_prompts = list(surv.questions)\n",
    "                dynamic_chunks_most_similar: List[List[str]] = []\n",
    "                progress = 0\n",
    "                lenn = len(dynamic_retrieval_prompts)\n",
    "                for prompt in dynamic_retrieval_prompts:\n",
    "                    progress += 1\n",
    "                    print(f\"\\rPrompt {progress}/{lenn}\", end=\"\")\n",
    "                    prompt_embedding = ollama.embeddings(model=EMBED_MODEL, prompt=prompt)[\"embedding\"]\n",
    "                    chunks_most_similar_embeddings_dynamic = utils.find_most_similar(prompt_embedding, embeddings)[:max_chunks_count // 2]\n",
    "                    chunks_most_similar_dynamic = [chunks[embedding[1]] for embedding in chunks_most_similar_embeddings_dynamic]\n",
    "                    dynamic_chunks_most_similar.append(chunks_most_similar_dynamic)\n",
    "                print(end=\"\\n\")\n",
    "\n",
    "                chunks_most_similar = chunks_most_similar_static + [chunk for sublist in dynamic_chunks_most_similar for chunk in sublist]\n",
    "                tokens_in_chunks = sum(utils.count_tokens(chunk) for chunk in chunks_most_similar)\n",
    "                del chunks_most_similar_embeddings_static, chunks_most_similar_embeddings_dynamic  # free memory\n",
    "\n",
    "                print(f\"Tokens in average chunk group: {tokens_in_chunks / len(chunks_most_similar)}\")\n",
    "                final_prompts = []\n",
    "                prompt_template = \"\"\"\n",
    "for question, chunks_most_similar_dynamic in zip(surv.questions, dynamic_chunks_most_similar):\n",
    "    p = [\n",
    "        systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famouns people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n**From most to least related**\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION RELATED TO THE SURVEY OVERALL:\\\\n\".join(chunks_most_similar_static),\n",
    "            \"\\\\n\\\\nNEW CONVERSATION RELATED TO THE PARTICULAR QUESTION:\\\\n\".join(chunks_most_similar_dynamic)\n",
    "        ])),      \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in an {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]\n",
    "    final_prompts.append(p)\n",
    "                \"\"\"\n",
    "            else: print(\"neither hybrid, nor dynamic\")\n",
    "            exec(prompt_template)\n",
    "            prompt_info = utils.describe_prompts_and_print(final_prompts) # Vanity print\n",
    "            SIM_ID = f\"{SUBJECT}-{WHICH_SURVEY}-{RETRIAVAL_METHOD}-{chunk_size}-{str(overlap_size).zfill(2)}-{str(max_chunks_count).zfill(2)}_{MODEL}_V6\"\n",
    "            bu.quickJSON(final_prompts, f\"data/5_monster_prep/{SIM_ID}_prompts.json\")\n",
    "            instructions = {\n",
    "                \"prompt_file\": f\"batch/prompts/{SIM_ID}_prompts.json\",\n",
    "                \"survey_type\": f\"{SURVEY_TYPE[0]}\",\n",
    "                \"isLocal\": True,\n",
    "                \"LIMIT\": None\n",
    "            }\n",
    "            settings = {\n",
    "                \"model\": MODEL,\n",
    "                \"timeout\": 300}\n",
    "            AUTO_INFO = {\n",
    "                \"CHUNK_SIZE\": chunk_size,\n",
    "                \"OVERLAP_SIZE\": overlap_size,\n",
    "                \"CHUNKS_COUNT_IN_CTX\": max_chunks_count,#chunks_count_in_ctx,\n",
    "                \"CTX_limit\": max_tokens,\n",
    "                \"tokens_in_chunks\": tokens_in_chunks,\n",
    "                \"model\": EMBED_MODEL,\n",
    "                \"prompt method\": PROMPT_METHOD,\n",
    "                \"retrieval method\": RETRIAVAL_METHOD,\n",
    "                \"retrieval prompt\": RETRIEVAL_PROMPT,\n",
    "                \"prompt_count\": PROMPT_COUNT,\n",
    "                \"survey\": WHICH_SURVEY,\n",
    "                \"subject\": SUBJECT,\n",
    "                \"prompt_template\": prompt_template,\n",
    "                **prompt_info,\n",
    "                **utils.describe_prompts([])\n",
    "            }\n",
    "            bu.quickJSON({\"instructions\": instructions, \"settings\": settings, \"info\": AUTO_INFO}, f\"data/5_monster_prep/batch-schema/{SIM_ID}_schema.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Display Info\n",
    "total_messages = sum(len(chat) for chat in et.selectedChats.values())\n",
    "chunks_count = len(chunks)\n",
    "avg_chunk_char_len = np.mean([len(chunk) for chunk in chunks])\n",
    "print(\n",
    "    f\"Chunk count: {chunks_count}\",\n",
    "    f\"Rough estimate of tokens per chunk: {round(avg_chunk_char_len / 4)} (4 characters per token)\",\n",
    "    f\"Messagees in input count: {total_messages}\",\n",
    "    f\"Messages in chunks count: {stat_total_msgs_in_chunks}\",\n",
    "    f\"Chunk \\ Input ratio: {round(stat_total_msgs_in_chunks / total_messages,2)} (OVERLAP_SIZE={OVERLAP_SIZE})\",\n",
    "    f\"Chunk Python type: {type(chunks[0])}\",\n",
    "    sep=\"\\n\"\n",
    ") \n",
    "bu.if_dir_not_exist_make(\"data/3_embeddings\")\n",
    "bu.quickJSON(AUTO_INFO, f\"data/3_embeddings/{EMBEDDING_ID}_info.json\")\n",
    "bu.quickJSON({\"chunks\": chunks, \"embeddings\": embeddings}, f\"data/3_embeddings/{EMBEDDING_ID}_embeddings.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display results\n",
    "# bu.if_dir_not_exist_make(\"data/4_chunks\")\n",
    "# bu.quickJSON(AUTO_INFO, f\"data/4_chunks/{CHECKPOINT}-dynamic_info.json\")\n",
    "# bu.quickJSON({\"chunks\": chunks, \"embeddings\": embeddings}, f\"data/4_chunks/{CHECKPOINT}-dynamic_embeddings.json\")\n",
    "############################################ VANITY BELOW ########################################\n",
    "tokens_in_chunks = 0\n",
    "for chunks_most_similar in dynamic_chunks_most_similar:\n",
    "    for chunk in chunks_most_similar:\n",
    "        tokens_in_chunks += utils.count_tokens(chunk)\n",
    "\n",
    "del chunks_most_similar_embeddings # free memory\n",
    "print(f\"Tokens in average chunk group: {tokens_in_chunks/len(dynamic_chunks_most_similar)}\")\n",
    "# bu.quickJSON(dynamic_chunks_most_similar, filename=f\"data/4_chunks/{CHECKPOINT}-dynamic_chunks.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "# Set the directory where the JSON files are located\n",
    "directory = 'batch/done/monster_7b'  # Replace with the actual directory path if needed\n",
    "\n",
    "# Loop through all files in the directory\n",
    "for filename in os.listdir(directory):\n",
    "    # Check if the file has a .json extension\n",
    "    if filename.endswith('.json'):\n",
    "        # Open the JSON file\n",
    "        file_path = os.path.join(directory, filename)\n",
    "        with open(file_path, 'r') as f:\n",
    "            data = json.load(f)\n",
    "\n",
    "        # Modify the \"model\" value\n",
    "        if \"settings\" in data and \"model\" in data[\"settings\"]:\n",
    "            data[\"settings\"][\"model\"] = \"llama3\"\n",
    "\n",
    "        # Write the modified data back to the file\n",
    "        with open(file_path, 'w') as f:\n",
    "            json.dump(data, f, indent=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}