airi_analysis_eli.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "import brikasutils as bu\n",
    "importlib.reload(bu)\n",
    "import shared_utils as utils\n",
    "from shared_utils import systemMsg, userMsg, assistantMsg\n",
    "importlib.reload(utils)\n",
    "import survey\n",
    "importlib.reload(survey)\n",
    "import persona\n",
    "importlib.reload(persona)\n",
    "\n",
    "import ollama\n",
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "from typing import List\n",
    "from openai import OpenAI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Analysis**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import survey\n",
    "import pandas as pd\n",
    "import re\n",
    "import importlib\n",
    "import shared_utils as utils\n",
    "importlib.reload(utils)\n",
    "import brikasutils as bu\n",
    "importlib.reload(bu)\n",
    "\n",
    "def see_if_column_valid(column_name, df, msg=\"Verification failed for\"):\n",
    "    dff = df[df[column_name].isna()]\n",
    "    dfff = dff.groupby(\"sim_signature\").apply(lambda x: x[x['run_number'] == 1], include_groups=False)\n",
    "    print(f\"{msg}: {len(dff)} ({len(dfff)} unique)\")\n",
    "    return dfff\n",
    "\n",
    "MOST_IMPORTANT_COLUMNS = ['sim_signature', 'run_number', \"model\", \"survey_type\", \"base_sim_signature\", \"SUBJECT\", 'CTX_limit', \"retrieval method\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Index all simulation files\n",
    "SIMULATIONS_DIR = \"analysis/sims-final-2\"\n",
    "\n",
    "sim_runs = []\n",
    "for root, dirs, files in os.walk(SIMULATIONS_DIR):\n",
    "    for file in files:\n",
    "        if file.split(\".\")[1] != \"json\":\n",
    "            print(f\"Invalid file (all must be json) {sim_run['path']}\")\n",
    "\n",
    "        sim_run = {}\n",
    "        sim_run[\"SIMULATION_ID\"] = file.split(\".\")[0]\n",
    "        sim_run[\"path\"] = os.path.join(root, file)\n",
    "        with open(sim_run[\"path\"], 'r') as f:\n",
    "            sim = json.load(f)\n",
    "        sim_run.update(sim[\"info\"][\"info\"])\n",
    "        sim_run.update(sim[\"info\"][\"settings\"])\n",
    "        sim_runs.append(sim_run)\n",
    "\n",
    "df = pd.DataFrame(sim_runs)\n",
    "df = df.dropna(axis=1, how='all')\n",
    "print(f\"Loaded {len(df)} simulation files\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Infer/Get Needed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_survey_type(row):\n",
    "    if \"survey_type\" in row and not pd.isna(row[\"survey_type\"]):\n",
    "        if row[\"survey_type\"] == \"KanoSurvey\":\n",
    "            return \"KanoSurvey\"\n",
    "        elif row[\"survey_type\"] == \"PersonalitySurvey\":\n",
    "            return \"PersonalitySurvey\"\n",
    "        else:\n",
    "            print(\"Unknown survey type\" + row[\"survey_type\"])\n",
    "\n",
    "    if \"prompt_count\" in row and not pd.isna(row[\"prompt_count\"]):\n",
    "        if row[\"prompt_count\"] == 50:\n",
    "            return \"PersonalitySurvey\"\n",
    "        elif row[\"prompt_count\"] == 40:\n",
    "            return \"KanoSurvey\"\n",
    "        else:\n",
    "            print(\"Unknown prompt count\" + row[\"prompt_count\"])\n",
    "\n",
    "    return None\n",
    "\n",
    "df[\"survey_type\"] = df.apply(infer_survey_type, axis=1) \n",
    "df[\"survey_type\"].value_counts()\n",
    "\n",
    "# extract_run_number\n",
    "def extract_run_number(sim_id):\n",
    "    try:\n",
    "        parts = sim_id.rsplit('_', 1)  # Attempt to split by the last underscore\n",
    "        if len(parts) == 2:  # Check if the split was successful\n",
    "            return pd.Series([parts[0], int(parts[-1])])\n",
    "        else:\n",
    "            print(f\"Error while processing {sim_id}\")\n",
    "            return pd.Series([pd.NA, pd.NA])  # Return None for last_number if split fails\n",
    "         \n",
    "    except Exception as e:  # Generic exception handling\n",
    "        print(f\"Error while processing {sim_id}\")\n",
    "        return pd.Series([pd.NA, pd.NA])\n",
    "\n",
    "df[['sim_signature', 'run_number']] = df['SIMULATION_ID'].apply(extract_run_number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# infer_if_simulation_is_base\n",
    "def infer_if_simulation_is_base(row):\n",
    "    if row[\"sim_signature\"][:4] == \"base\":\n",
    "        return True\n",
    "    return False\n",
    "    \n",
    "df[\"is_base\"] = df.apply(infer_if_simulation_is_base, axis=1)\n",
    "\n",
    "# Below: Vanity Print\n",
    "dff = df[df[\"is_base\"] == True]\n",
    "dff = dff.sort_values(by=['sim_signature', 'run_number'])\n",
    "dff = dff.dropna(axis=1, how='all')\n",
    "dfff = dff.groupby(\"sim_signature\").apply(lambda x: x[x['run_number'] == 1], include_groups=False)\n",
    "print(f\"Found {len(dff)} ({len(dfff)} unique) base simulations\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preview base\n",
    "dfff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Map simulations to their base\n",
    "def map_simulation_to_base(row):\n",
    "    if row[\"is_base\"] == True:\n",
    "        return \"(base)\"\n",
    "    if row[\"survey_type\"] == \"KanoSurvey\" and row[\"model\"] == \"gpt-3.5-turbo\":\n",
    "        return \"base_kano_v2_gpt35\"\n",
    "\n",
    "    if row[\"survey_type\"] == \"PersonalitySurvey\" and row[\"model\"] == \"gpt-3.5-turbo\":\n",
    "        return \"base_personality_v2_gpt35\"\n",
    "\n",
    "    if row[\"survey_type\"] == \"KanoSurvey\" and row[\"model\"] == \"llama3-70b\":\n",
    "        return \"base-kano-29_llama3-70b_V7\"\n",
    "\n",
    "    if row[\"survey_type\"] == \"PersonalitySurvey\" and row[\"model\"] == \"llama3-70b\":\n",
    "        return \"base-pers-29_llama3-70b_V7\"\n",
    "    \n",
    "    if row[\"survey_type\"] == \"KanoSurvey\" and row[\"model\"] == \"llama3-8b\":\n",
    "        return \"base-kano-29_llama3-8b_V7\"\n",
    "\n",
    "    if row[\"survey_type\"] == \"PersonalitySurvey\" and row[\"model\"] == \"llama3-8b\":\n",
    "        return \"base-pers-29_llama3-8b_V7\"\n",
    "    \n",
    "    if row[\"survey_type\"] == \"KanoSurvey\" and row[\"model\"] == \"mixtral-8x22b\":\n",
    "        return \"base-kano-29_mixtral-8x22b_V7\"\n",
    "\n",
    "    if row[\"survey_type\"] == \"PersonalitySurvey\" and row[\"model\"] == \"mixtral-8x22b\":\n",
    "        return \"base-pers-29_mixtral-8x22b_V7\"\n",
    "    \n",
    "    return pd.NA\n",
    "    \n",
    "df[\"base_sim_signature\"] = df.apply(map_simulation_to_base, axis=1)\n",
    "dff = see_if_column_valid(\"base_sim_signature\", df, \"Missing mappings\")\n",
    "if len(dff) == 0:\n",
    "    print(\"All mappings are valid\")\n",
    "else:\n",
    "    print(\"Not all mappings are valid. See the missing mappings below\")\n",
    "    display(dff)\n",
    "\n",
    "def infer_subject(row):\n",
    "    if row[\"is_base\"]:\n",
    "        return \"(base)\"\n",
    "    if \"SUBJECT\" in row and pd.notna(row[\"SUBJECT\"]):\n",
    "        if row[\"SUBJECT\"] == \"airidas\" or row[\"SUBJECT\"] == \"Airidas\" or row[\"SUBJECT\"] == \"airi\":\n",
    "            return \"airidas\"\n",
    "        if row[\"SUBJECT\"] == \"elias\" or row[\"SUBJECT\"] == \"eli\":\n",
    "            return \"elias\"\n",
    "        print(f\"Unknown subject: {row['SUBJECT']}\")\n",
    "        return pd.NA\n",
    "    if \"subject\" in row and pd.notna(row[\"subject\"]):\n",
    "        if row[\"subject\"] == \"airidas\" or row[\"subject\"] == \"Airidas\" or row[\"subject\"] == \"airi\":\n",
    "            return \"airidas\"\n",
    "        if row[\"subject\"] == \"elias\" or row[\"subject\"] == \"eli\":\n",
    "            return \"elias\"\n",
    "        print(f\"Unknown subject: {row['subject']}\")\n",
    "        return pd.NA\n",
    "    if row[\"sim_signature\"][:4] == \"airi\":\n",
    "        return \"airidas\"\n",
    "    if row[\"sim_signature\"][:3] == \"eli\":\n",
    "        return \"elias\"\n",
    "    return pd.NA\n",
    "\n",
    "df[\"SUBJECT\"] = df.apply(infer_subject, axis=1)\n",
    "dff = see_if_column_valid(\"SUBJECT\", df, \"Missing subjects\")\n",
    "if len(dff) == 0:\n",
    "    print(\"All subjects are valid\")\n",
    "else:\n",
    "    display(dff)\n",
    "\n",
    "df = utils.bring_to_front_important_columns(df, MOST_IMPORTANT_COLUMNS)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "er = [] # Extraction records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_surv_from_info(row):\n",
    "    if row[\"survey_type\"] == \"KanoSurvey\":\n",
    "        return survey.KanoSurvey()\n",
    "    elif row[\"survey_type\"] == \"PersonalitySurvey\":\n",
    "        return survey.PersonalitySurvey()\n",
    "    else:\n",
    "        raise Exception(\"Unknown survey type: \" + row[\"survey_type\"])\n",
    "\n",
    "all_possible_asnwers = [\"I LIKE IT\", \"I EXPECT IT\", \"I AM NEUTRAL\", \"I CAN TOLERATE IT\", \"I DISLIKE IT\", \"SOMEWHAT DISAGREE\", \"DISAGREE\", \"NEUTRAL\", \"SOMEWHAT AGREE\", \"AGREE\"]\n",
    "def extract_possible_answer(value):\n",
    "    for phrase in all_possible_asnwers:\n",
    "        pattern = r'(?i)' + re.escape(phrase)\n",
    "        match = re.search(pattern, value)\n",
    "        if match:\n",
    "            # if value != phrase:\n",
    "            #     er.append([value, phrase])  \n",
    "            return match.group()\n",
    "    return value  # Return the original value if no possible answer is found\n",
    "\n",
    "############ Invalid Answers ##################\n",
    "def get_invalid_answers(value):\n",
    "    if pd.isna(value):\n",
    "        return \"\"\n",
    "    elif value == \"NaN\":\n",
    "        return \"\"\n",
    "    elif value in all_possible_asnwers:\n",
    "        return \"\"\n",
    "    else:\n",
    "        return value\n",
    "\n",
    "def clean_simulation_QA(df) -> pd.DataFrame:\n",
    "    df['answer'] = df['answer'].apply(lambda x: x.strip())\n",
    "    for substr in utils.BLACKLIST_ANSWER_SUBSTRINGS:\n",
    "        df['answer'] = df['answer'].apply(lambda x: re.sub(substr, \"\", x))\n",
    "    df['answer'] = df['answer'].str.upper()\n",
    "    df['answer'] = df['answer'].apply(extract_possible_answer)\n",
    "\n",
    "    REMAP_MISSING_E = {\"AGRE\": \"AGREE\", \"SOMEWHAT AGRE\": \"SOMEWHAT AGREE\", \"SOMEWHAT DISAGRE\": \"SOMEWHAT DISAGREE\", \"DISAGRE\": \"DISAGREE\", \"I DON'T LIKE IT\": \"I DISLIKE IT\"}\n",
    "    df.iloc[:, 1:] = df.iloc[:, 1:].map(lambda x: REMAP_MISSING_E.get(x, x))\n",
    "\n",
    "    PARTIAL_MATCH_REMAP = {\"SOMEWHAT AG\":\"SOMEWHAT AGREE\" }\n",
    "    df[\"answer\"] = df[\"answer\"].apply(\n",
    "        lambda answer: next((value for key, value in PARTIAL_MATCH_REMAP.items() if key in answer), answer)\n",
    "    )\n",
    "\n",
    "    # Update isValid\n",
    "    df['isValid'] = df['answer'].apply(lambda x: x in all_possible_asnwers)\n",
    "\n",
    "    # if all values in isValid is true, drop the column, else print a message\n",
    "    if not df['isValid'].all():\n",
    "        print(\"Warning, some answers were not valid. See df['isValid']\")      \n",
    "\n",
    "    return df\n",
    "\n",
    "# Proces simulation output\n",
    "def add_airidas_and_elias_answers(df, surv) -> pd.DataFrame:\n",
    "    # Add airidas and elias answers\n",
    "    air = surv.test_answers[\"airidas\"]\n",
    "    eli = surv.test_answers[\"elias\"]\n",
    "\n",
    "    # Sanity Check\n",
    "    if len(air) != len(df):\n",
    "        raise Exception(f\"Survey and DF length mismatch {len(air)} != {len(df)}. Suvey type: {str(type(surv))}\")\n",
    "\n",
    "    df.insert(2, \"airidas\", air[:len(df)])\n",
    "    df.insert(3, \"elias\", eli[:len(df)])\n",
    "\n",
    "    # Convert to uppercase\n",
    "    if isinstance(surv, survey.KanoSurvey):\n",
    "        df['answer'] = df['answer'].str.upper()\n",
    "        df['airidas'] = df['airidas'].str.upper()\n",
    "        df['elias'] = df['elias'].str.upper()\n",
    "        \n",
    "    return df\n",
    "\n",
    "def remap_answers_to_integers(df, surv):\n",
    "    if isinstance(surv, survey.KanoSurvey):\n",
    "        remap_dict = {\"I EXPECT IT\": 5, \"I LIKE IT\": 4, \"I AM NEUTRAL\": 3, \"I CAN TOLERATE IT\": 2, \"I DISLIKE IT\": 1}\n",
    "        df['answer'] = df['answer'].map(remap_dict)\n",
    "        df['airidas'] = df['airidas'].map(remap_dict)\n",
    "        df['elias'] = df['elias'].map(remap_dict)\n",
    "    elif isinstance(surv, survey.PersonalitySurvey):\n",
    "        remap_dict = {\"AGREE\": 5, \"SOMEWHAT AGREE\": 4, \"NEUTRAL\": 3, \"SOMEWHAT DISAGREE\": 2, \"DISAGREE\": 1}\n",
    "        df['answer'] = df['answer'].map(remap_dict)\n",
    "\n",
    "    return df\n",
    "\n",
    "def evaluate_single_simulation_run(df) -> dict:\n",
    "    # compute the percentage of correct answers and average loss (MAE)\n",
    "    result_data = {\n",
    "        \"p-corr_Airidas\": df['answer'].corr(df['airidas']),\n",
    "        \"p-corr_Elias\": df['answer'].corr(df['elias']),\n",
    "        # Average of absolute residuals for Airidas\n",
    "        \"MAE_airi\": (df['answer'] - df['airidas']).abs().sum() / len(df),\n",
    "        # Average of absolute residuals for Elias\n",
    "        \"MAE_eli\": (df['answer'] - df['elias']).abs().sum() / len(df),\n",
    "        \"question_count\": len(df),\n",
    "    }\n",
    "    return result_data\n",
    "\n",
    "\n",
    "ADD_TO_MOST_IMPORTANT_COLUMNS = [\"MAE_airi\", \"MAE_eli\"]\n",
    "for col in ADD_TO_MOST_IMPORTANT_COLUMNS:\n",
    "    if col not in MOST_IMPORTANT_COLUMNS:\n",
    "        MOST_IMPORTANT_COLUMNS.append(col)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Multi-Sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = df[df['model'] != 'mixtral-8x22b']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EVALUATE_INVALID_SIMULATIONS = True\n",
    "invalid_vals = []\n",
    "tmp = []\n",
    "\n",
    "for index, sim_row in df.iterrows():\n",
    "    sim = utils.load_sim(sim_row[\"path\"])\n",
    "    dfQA = utils.dataframe_from_QA(sim[\"QA\"])\n",
    "    with bu.MutePrint():\n",
    "        surv = get_surv_from_info(sim_row)\n",
    "        dfQA = clean_simulation_QA(dfQA)\n",
    "    dfQA = add_airidas_and_elias_answers(dfQA, surv)\n",
    "\n",
    "    ## Check for invalid values\n",
    "    if not dfQA['isValid'].all():\n",
    "        invalid_vals.extend(dfQA.loc[~dfQA['isValid'], 'answer'].tolist())\n",
    "        \n",
    "        if EVALUATE_INVALID_SIMULATIONS:\n",
    "            print(f\"{sim_row['SIMULATION_ID']} has invalid value(s). Sim will be included with dropped rows.\")\n",
    "            dfQA = dfQA[dfQA['isValid'] == True]\n",
    "        else:\n",
    "            print(f\"Skipping {sim_row['SIMULATION_ID']} due to invalid answers\")\n",
    "            continue\n",
    "\n",
    "\n",
    "    dfQA = remap_answers_to_integers(dfQA, surv)\n",
    "    res = evaluate_single_simulation_run(dfQA)\n",
    "\n",
    "    for key, value in res.items():\n",
    "        df.at[index, key] = round(value, 3)\n",
    "\n",
    "# Rename values\n",
    "df['CTX_limit'] = df['CTX_limit'].astype(str)\n",
    "df.loc[df['CTX_limit'] == 0, 'CTX_limit'] = '1-chunk'\n",
    "\n",
    "df = utils.bring_to_front_important_columns(df, MOST_IMPORTANT_COLUMNS)\n",
    "\n",
    "if len(invalid_vals) > 0:\n",
    "    print(f\"{len(invalid_vals)} Invalid values:\")\n",
    "    display(pd.DataFrame(invalid_vals, columns=[\"Invalid Values\"]))\n",
    "    if EVALUATE_INVALID_SIMULATIONS:\n",
    "        print(\"EVALUATE_INVALID_SIMULATIONS == True. All invalid values were dropped\")\n",
    "else:\n",
    "    print(\"All values are valid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define how columns should be groped\n",
    "aggregation_dict = {\n",
    "\n",
    "    'MAE_airi': ['mean', 'std'],\n",
    "    'MAE_eli': ['mean', 'std'],\n",
    "    'run_number': ['count'],\n",
    "    'question_count': ['min'],\n",
    "    'p-corr_Airidas': ['mean', 'std'],\n",
    "    'p-corr_Elias': ['mean', 'std'],\n",
    "}\n",
    "# Preserve the first entry of other columns\n",
    "for col in df.columns:\n",
    "    if col not in ['sim_signature', *list(aggregation_dict.keys())]:\n",
    "        aggregation_dict[col] = 'first'\n",
    "        \n",
    "# dfg stands for DataFrame Grouped.\n",
    "dfg = df.groupby('sim_signature').agg(aggregation_dict)\n",
    "\n",
    "# Renaming MultiIndex columns\n",
    "dfg.columns = ['_'.join(col).strip() if col[1] != 'first' else col[0] for col in dfg.columns.values]\n",
    "dfg.rename(columns={'run_number_count': 'n-runs'}, inplace=True)\n",
    "\n",
    "dfg = dfg.reset_index()\n",
    "print(f\"Total unique simulations: {len(dfg)}\")\n",
    "dfg.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vizualisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Assuming 'mean_residual_Airidas_mean' and 'p-corr_Airidas_mean' are already computed as mean values in your aggregated dataframe\n",
    "# Plotting for Airidas\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.scatter(dfg['p-corr_Airidas_mean'], dfg['MAE_airi_mean'], label='Airidas', alpha=0.5)\n",
    "\n",
    "# Assuming 'mean_residual_Elias_mean' and 'p-corr_Elias_mean' are also computed as mean values\n",
    "# Plotting for Elias\n",
    "plt.scatter(dfg['p-corr_Elias_mean'], dfg['MAE_eli_mean'], color='red', label='Elias', alpha=0.5)\n",
    "\n",
    "plt.title('Mean Residuals vs P-Corr')\n",
    "plt.xlabel('P-Corr (mean)')\n",
    "plt.ylabel('Mean Residuals (mean)')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "\n",
    "# Set style\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "# Sorting and filtering the dataframe for each plot\n",
    "sorted_airidas_personality = dfg[(dfg['survey_type'] == 'PersonalitySurvey')].sort_values('MAE_airi_mean', ascending=True)\n",
    "sorted_airidas_kano = dfg[(dfg['survey_type'] == 'KanoSurvey')].sort_values('MAE_airi_mean', ascending=True)\n",
    "sorted_elias_personality = dfg[(dfg['survey_type'] == 'PersonalitySurvey')].sort_values('MAE_eli_mean', ascending=True)\n",
    "sorted_elias_kano = dfg[(dfg['survey_type'] == 'KanoSurvey')].sort_values('MAE_eli_mean', ascending=True)\n",
    "\n",
    "# Custom color functions\n",
    "def get_colors_airidas(df):\n",
    "    colors = []\n",
    "    for _, row in df.iterrows():\n",
    "        if row['SUBJECT'] != 'airidas' and not row['is_base']:\n",
    "            colors.append('#bfd7ec')\n",
    "        elif row['is_base']:\n",
    "            colors.append('black')\n",
    "        else:\n",
    "            colors.append('#0c4da2')\n",
    "    return colors\n",
    "\n",
    "def get_colors_elias(df):\n",
    "    colors = []\n",
    "    for _, row in df.iterrows():\n",
    "        if row['SUBJECT'] != 'elias' and not row['is_base']:\n",
    "            colors.append('#adcbe3')\n",
    "        elif row['is_base']:\n",
    "            colors.append('black')\n",
    "        else:\n",
    "            colors.append('#0c4da2')\n",
    "    return colors\n",
    "\n",
    "colors_airidas_p = get_colors_airidas(sorted_airidas_personality)\n",
    "colors_airidas_k = get_colors_airidas(sorted_airidas_kano)\n",
    "colors_elias_p = get_colors_elias(sorted_elias_personality)\n",
    "colors_elias_k = get_colors_elias(sorted_elias_kano)\n",
    "\n",
    "fig, axs = plt.subplots(2, 2, figsize=(14, 14))\n",
    "fig.suptitle('Mean MAE Metrics by Sim Signature and Survey Type', fontsize=16)\n",
    "\n",
    "# Adding horizontal bars and error bars\n",
    "for (data, ax, colors, ylabel, title) in [\n",
    "    (sorted_airidas_personality, axs[0, 0], colors_airidas_p, 'sim_signature', 'Mean MAE_Airidas (PersonalitySurvey)'),\n",
    "    (sorted_airidas_kano, axs[1, 0], colors_airidas_k, 'sim_signature', 'Mean MAE_Airidas (KanoSurvey)'),\n",
    "    (sorted_elias_personality, axs[0, 1], colors_elias_p, 'sim_signature', 'Mean MAE_Elias (PersonalitySurvey)'),\n",
    "    (sorted_elias_kano, axs[1, 1], colors_elias_k, 'sim_signature', 'Mean MAE_Elias (KanoSurvey)')\n",
    "]:\n",
    "    sns.barplot(data=data, y=ylabel, x='MAE_airi_mean' if 'Airidas' in title else 'MAE_eli_mean', ax=ax, palette=colors, orient='h')\n",
    "    if 'Airidas' in title:\n",
    "        ax.errorbar(data['MAE_airi_mean'], data[ylabel], xerr=data['MAE_airi_std'], fmt='none', ecolor='red', capsize=3, elinewidth=1, alpha=1 )\n",
    "    else:\n",
    "        ax.errorbar(data['MAE_eli_mean'], data[ylabel], xerr=data['MAE_eli_std'], fmt='none', ecolor='red', capsize=3, elinewidth=1, alpha=1)\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(title.split(' ')[1])\n",
    "    ax.set_ylabel(ylabel)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
    "plt.show()\n",
    "\n",
    "###########################\n",
    "# Average the crossed results\n",
    "###########################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# expoert dfg to csv\n",
    "dfg.to_csv(\"analysis/spreadsheets/dfg-1.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning analysis (NEW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "\n",
    "# Assuming 'dfg' is your DataFrame\n",
    "# Filter out rows where 'is_base' is True\n",
    "df_filtered = dfg[dfg['is_base'] == False]\n",
    "\n",
    "# Create a figure for the plots\n",
    "fig, axs = plt.subplots(2, 2, figsize=(14, 12))  # 2 rows, 2 columns\n",
    "fig.suptitle('Comparison of Models for Static vs Dynamic (Lower is better)', fontsize=16)\n",
    "\n",
    "# Each subplot for different combinations\n",
    "for i, (subject, survey_type) in enumerate([\n",
    "        ('elias', 'KanoSurvey'), ('elias', 'PersonalitySurvey'),\n",
    "        ('airidas', 'KanoSurvey'), ('airidas', 'PersonalitySurvey')]):\n",
    "\n",
    "    ax = axs[i//2, i%2]\n",
    "    \n",
    "    # Filter data for the specific group\n",
    "    df_group = df_filtered[(df_filtered['SUBJECT'] == subject) & (df_filtered['survey_type'] == survey_type)]\n",
    "    \n",
    "    # Separate further by model\n",
    "    df_group_8b = df_group[df_group['model'] == 'llama3-8b']\n",
    "    df_group_70b = df_group[df_group['model'] == 'llama3-70b']\n",
    "    df_group_8x22b = df_group[df_group['model'] == 'mixtral-8x22b']\n",
    "    \n",
    "    # Combine the data for a comparison by retrieval method and model\n",
    "    df_group_8b = df_group_8b.assign(Model='lamma3-8b')\n",
    "    df_group_70b = df_group_70b.assign(Model='lamma3-70b')\n",
    "    df_group_8x22b = df_group_8x22b.assign(Model='mixtral-8x22')\n",
    "    df_plot = pd.concat([df_group_8b, df_group_70b, df_group_8x22b])\n",
    "\n",
    "    # Select the correct metric based on the subject\n",
    "    if subject == 'elias':\n",
    "        metric_column = f'MAE_eli_mean'\n",
    "    if subject == 'airidas':\n",
    "        metric_column = f'MAE_airi_mean'\n",
    "    df_plot['Metric Value'] = df_plot[metric_column]\n",
    "    \n",
    "    # Create a bar chart\n",
    "    sns.barplot(data=df_plot, x='retrieval method', y='Metric Value', hue='Model', ax=ax, palette='viridis')\n",
    "    \n",
    "    # Setting the title and labels\n",
    "    ax.set_title(f'{subject.capitalize()} - {survey_type}')\n",
    "    ax.set_xlabel('Retrieval Method')\n",
    "    ax.set_ylabel('Metric Value')\n",
    "\n",
    "# Adjust layout for better readability\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
    "\n",
    "# Display the plot\n",
    "plt.show()\n",
    "\n",
    "\n",
    "##########################\n",
    "# Add base\n",
    "#########################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "\n",
    "# Assuming 'dfg' is your DataFrame\n",
    "# Filter out rows where 'is_base' is True and 'lamma3-70b'\n",
    "\n",
    "df_filtered = dfg[(dfg['is_base'] == False) & (dfg['model'] == 'llama3-8b')]\n",
    "\n",
    "# Create a figure for the plots\n",
    "fig, axs = plt.subplots(2, 2, figsize=(14, 12))  # 2 rows, 2 columns\n",
    "fig.suptitle('Comparison of CTX Limit Values for llama3-8b', fontsize=16)\n",
    "\n",
    "# Each subplot for different combinations\n",
    "for i, (subject, survey_type) in enumerate([\n",
    "        ('elias', 'KanoSurvey'), ('elias', 'PersonalitySurvey'),\n",
    "        ('airidas', 'KanoSurvey'), ('airidas', 'PersonalitySurvey')]):\n",
    "\n",
    "    ax = axs[i//2, i%2]\n",
    "    \n",
    "    # Filter data for the specific group\n",
    "    df_group = df_filtered[(df_filtered['SUBJECT'] == subject) & (df_filtered['survey_type'] == survey_type)]\n",
    "    \n",
    "    # Select the correct metric based on the subject\n",
    "    metric_column = f'p-corr_{subject.capitalize()}_mean'\n",
    "    df_group['Metric Value'] = df_group[metric_column]\n",
    "    \n",
    "    # Create a bar chart\n",
    "    sns.barplot(data=df_group, x='CTX_limit', y='Metric Value', hue='retrieval method', ax=ax, palette= \"inferno\")\n",
    "    \n",
    "    # Setting the title and labels\n",
    "    ax.set_title(f'{subject.capitalize()} - {survey_type}')\n",
    "    ax.set_xlabel('CTX Limit')\n",
    "    ax.set_ylabel('Metric Value')\n",
    "\n",
    "# Adjust layout for better readability\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
    "\n",
    "# Display the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "dfg = dfg[dfg['SUBJECT'] != \"airidas\"]\n",
    "dfg = dfg[dfg['retrieval method'] == \"dynamic\"]\n",
    "# Compute mean p-corr_Elias_mean for each unique value of 'CHUNKS_COUNT_IN_CTX'\n",
    "chunks_count_means = dfg.groupby('CHUNKS_COUNT_IN_CTX')['p-corr_Elias_mean'].mean().reset_index()\n",
    "chunks_count_means = chunks_count_means.sort_values(by='CHUNKS_COUNT_IN_CTX')\n",
    "\n",
    "# Plot bar chart for 'CHUNKS_COUNT_IN_CTX'\n",
    "plt.figure(figsize=(15, 6))\n",
    "plt.bar(np.arange(len(chunks_count_means)), chunks_count_means['p-corr_Elias_mean'])\n",
    "plt.xticks(np.arange(len(chunks_count_means)), chunks_count_means['CHUNKS_COUNT_IN_CTX'], rotation=90, ha='right')\n",
    "plt.subplots_adjust(bottom=0.3)\n",
    "plt.ylabel('Mean p-corr_Elias_mean')\n",
    "plt.title('Mean p-corr_Elias_mean by CHUNKS_COUNT_IN_CTX')\n",
    "plt.show()\n",
    "\n",
    "# Compute mean p-corr_Elias_mean for each unique value of 'CHUNK_SIZE'\n",
    "chunk_size_means = dfg.groupby('CHUNK_SIZE')['p-corr_Elias_mean'].mean().reset_index()\n",
    "chunk_size_means = chunk_size_means.sort_values(by='CHUNK_SIZE')\n",
    "\n",
    "# Plot bar chart for 'CHUNK_SIZE'\n",
    "plt.figure(figsize=(15, 6))\n",
    "plt.bar(np.arange(len(chunk_size_means)), chunk_size_means['p-corr_Elias_mean'])\n",
    "plt.xticks(np.arange(len(chunk_size_means)), chunk_size_means['CHUNK_SIZE'], rotation=90, ha='right')\n",
    "plt.subplots_adjust(bottom=0.3)\n",
    "plt.ylabel('Mean p-corr_Elias_mean')\n",
    "plt.title('Mean p-corr_Elias_mean by CHUNK_SIZE')\n",
    "plt.show()\n",
    "\n",
    "# Compute mean p-corr_Elias_mean for each unique value of 'OVERLAP_SIZE'\n",
    "overlap_size_means = dfg.groupby('OVERLAP_SIZE')['p-corr_Elias_mean'].mean().reset_index()\n",
    "overlap_size_means = overlap_size_means.sort_values(by='OVERLAP_SIZE')\n",
    "\n",
    "# Plot bar chart for 'OVERLAP_SIZE'\n",
    "plt.figure(figsize=(15, 6))\n",
    "plt.bar(np.arange(len(overlap_size_means)), overlap_size_means['p-corr_Elias_mean'])\n",
    "plt.xticks(np.arange(len(overlap_size_means)), overlap_size_means['OVERLAP_SIZE'], rotation=90, ha='right')\n",
    "plt.subplots_adjust(bottom=0.3)\n",
    "plt.ylabel('Mean p-corr_Elias_mean')\n",
    "plt.title('Mean p-corr_Elias_mean by OVERLAP_SIZE')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Drop rows where SUBJECT == \"airidas\"\n",
    "dfg = dfg[dfg['SUBJECT'] != \"airidas\"]\n",
    "dfg = dfg[dfg['retrieval method'] == \"dynamic\"]\n",
    "# dfg = dfg[dfg['survey_type'] != \"PersonaSurvey\"]\n",
    "\n",
    "# Compute mean p-corr_Elias_mean for each unique value of 'retrieval_method'\n",
    "retrieval_method_means = dfg.groupby('retrieval method')['p-corr_Elias_mean'].mean().reset_index()\n",
    "retrieval_method_means = retrieval_method_means.sort_values(by='retrieval method')\n",
    "\n",
    "# Plot bar chart for 'retrieval_method'\n",
    "plt.figure(figsize=(15, 6))\n",
    "plt.bar(np.arange(len(retrieval_method_means)), retrieval_method_means['p-corr_Elias_mean'])\n",
    "plt.xticks(np.arange(len(retrieval_method_means)), retrieval_method_means['retrieval method'], rotation=90, ha='right')\n",
    "plt.subplots_adjust(bottom=0.3)\n",
    "plt.ylabel('Mean p-corr_Elias_mean')\n",
    "plt.title('Mean p-corr_Elias_mean by Retrieval Method')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "# Drop rows where SUBJECT == \"airidas\"\n",
    "dfg = dfg[dfg['SUBJECT'] != \"airidas\"]\n",
    "\n",
    "\n",
    "# Compute mean p-corr_Elias_mean and mean_residual_Elias_std for each unique value of 'retrieval_method'\n",
    "retrieval_method_means = dfg.groupby('retrieval method')[['p-corr_Elias_mean', 'mean_residual_Elias_std']].mean().reset_index()\n",
    "retrieval_method_means = retrieval_method_means.sort_values(by='retrieval method')\n",
    "\n",
    "# Plot bar chart for 'retrieval_method'\n",
    "fig, ax = plt.subplots(figsize=(15, 6))\n",
    "bar_width = 0.35\n",
    "index = np.arange(len(retrieval_method_means))\n",
    "\n",
    "rects1 = ax.bar(index, retrieval_method_means['p-corr_Elias_mean'], bar_width, label='p-corr_Elias_mean')\n",
    "rects2 = ax.bar(index + bar_width, retrieval_method_means['mean_residual_Elias_std'], bar_width, label='mean_residual_Elias_std')\n",
    "\n",
    "ax.set_xlabel('Retrieval Method')\n",
    "ax.set_ylabel('Mean Value')\n",
    "ax.set_title('Mean Values by Retrieval Method')\n",
    "ax.set_xticks(index + bar_width / 2)\n",
    "ax.set_xticklabels(retrieval_method_means['retrieval method'], rotation=90, ha='right')\n",
    "ax.legend()\n",
    "\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# Drop rows where SUBJECT == \"airidas\"\n",
    "# dfg = dfg[dfg['SUBJECT'] != \"airidas\"]\n",
    "# Compute summary statistics for each retrieval method\n",
    "summary_stats = dfg.groupby('retrieval method')[['p-corr_Elias_mean', 'mean_residual_Elias_std']].agg(['mean', 'var', 'std'])\n",
    "# Reset the multi-index to make the column labels more readable\n",
    "summary_stats.columns = ['_'.join(col).strip() for col in summary_stats.columns.values]\n",
    "summary_stats = summary_stats.reset_index()\n",
    "# Store the summary statistics in a DataFrame\n",
    "pd.DataFrame(summary_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "systemMsg( \"You are participating in a survey. You will be presented with a series of questions about your video game preferrences.\", f\"You must choose answer to the question below with one of the five options: {', '.join(surv.POSSIBLE_ANSWERS)}. The answer must only contain the chosen option. \" ), \n",
    "# Understanding affirmation \n",
    "assistantMsg('Understood. I will answer the question below with one of the given options.'), \n",
    "# Survey question. With Simulation \n",
    "userMsg( question, \"Your choice: \" ),"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "systemMsg(\"\\\\n\".join([\n",
    "            f\"You are an expert actor, specializing in impersonation of non-famouns people. You will be presented to the subject through explicit datapoints of their digital footprint. In addition, you will deduct their implicit {SURVEY} by shadowing chats between the subject and friends. You will be asked to fully immerse yourself in the role, and answer questions from the point of view of the persona. \\\\n#Context \\\\n##Chat conversations between the subject and their friends:\\\\n**From most to least related**\\\\n\",\n",
    "            \"\\\\n\\\\nNEW CONVERSATION:\\\\n\".join(chunks_most_similar)\n",
    "        ])),      \n",
    "        assistantMsg(\"Understood. I will answer from the point of view of the persona, based on what I could the deduct from the text provided.\"),\n",
    "        userMsg(\"\\\\n\".join([\n",
    "            f\"Persona is questioned about their {SURVEY} in an {METHOD}. The persona must choose an appropriate answer to the question below with one of these five given options: {', '.join(surv.POSSIBLE_ANSWERS)}. Persona's answer must only contain the chosen option, without any elaboration, nor introduction.\\\\n\\\\n**Your question is:**\\\\n\",\n",
    "            question,\n",
    "            \"\\\\nThe persona chooses:\"\n",
    "        ]))]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}