run_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of ChroKnowPrompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sources.utils import *\n",
    "from sources.process import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Please follow the order down below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Get Year Span Information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-1. Load data with chrono ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data_with_updated_timestamp(model_name, domain):\n",
    "    bench_dir_c = f\"./ChroKnowBench/TimeVariant_{domain}_Dynamic.jsonl\"\n",
    "    bench_dir_u = f\"./ChroKnowBench/TimeVariant_{domain}_Static.jsonl\"\n",
    "    timestamp_dir_c = f'./ChronoGap/{model_name}/Updated_Timestamp_{domain}_Dynamic.json'\n",
    "    timestamp_dir_u = f'./ChronoGap/{model_name}/Updated_Timestamp_{domain}_Static.json'\n",
    "    bench_data_c = read_jsonl_file(bench_dir_c)\n",
    "    bench_data_u = read_jsonl_file(bench_dir_u)\n",
    "    timestamp_c = read_json_file(timestamp_dir_c)\n",
    "    timestamp_u = read_json_file(timestamp_dir_u)\n",
    "    return bench_data_c, bench_data_u, timestamp_c, timestamp_u\n",
    "\n",
    "def get_traversal_details(partial_known, target_year, tentative_ans, prev_year_span=3, next_year_span=3):\n",
    "    \"\"\"\n",
    "    Determines the traversal steps taken to locate the tentative answer within the specified spans.\n",
    "\n",
    "    Parameters:\n",
    "    - partial_known (dict): The known data containing answers categorized by years.\n",
    "    - target_year (str): The target year key (e.g., 'objects_2022').\n",
    "    - tentative_ans (str): The tentative answer to be checked.\n",
    "    - prev_year_span (int): Number of previous years to consider.\n",
    "    - next_year_span (int): Number of next years to consider.\n",
    "\n",
    "    Returns:\n",
    "    - dict: A dictionary containing traversal steps.\n",
    "    \"\"\"\n",
    "\n",
    "    def extract_year_from_key(key):\n",
    "        \"\"\"Extracts the year from keys like 'objects_2021'.\"\"\"\n",
    "        return int(key.split('_')[1])\n",
    "\n",
    "    def get_span_years(partial_known, target_year, direction, span_limit):\n",
    "        \"\"\"Returns a list of years within the specified span and direction.\"\"\"\n",
    "        target_year_int = extract_year_from_key(target_year)\n",
    "        years = sorted([extract_year_from_key(key) for key in partial_known.keys()])\n",
    "        if direction == 'previous':\n",
    "            span_years = [year for year in years if year < target_year_int][-span_limit:]\n",
    "        else:\n",
    "            span_years = [year for year in years if year > target_year_int][:span_limit]\n",
    "        return span_years\n",
    "\n",
    "    def collect_answers_by_year(partial_known, span_years):\n",
    "        \"\"\"Collects answers for each year in span_years.\"\"\"\n",
    "        answers_by_year = {}\n",
    "        for year in span_years:\n",
    "            year_key = f'objects_{year}'\n",
    "            reference_data = partial_known.get(year_key, {})\n",
    "            answers = set()\n",
    "            if \"chrono_ans\" in reference_data:\n",
    "                chrono_ans = reference_data[\"chrono_ans\"]\n",
    "                if isinstance(chrono_ans, list):\n",
    "                    answers.update([ans.lower() for ans in chrono_ans if ans])\n",
    "                else:\n",
    "                    answers.add(chrono_ans.lower())\n",
    "            elif reference_data.get(\"category\") == \"correct\":\n",
    "                temp0_ans = reference_data.get(\"temp0_ans\", [])\n",
    "                answers.update([ans.lower() for ans in temp0_ans if ans])\n",
    "            else:\n",
    "                temp7_ans = reference_data.get(\"temp7_ans\", [])\n",
    "                answers.update([ans.lower() for ans in temp7_ans if ans])\n",
    "            answers_by_year[year] = answers\n",
    "        return answers_by_year\n",
    "\n",
    "    # Step 1: Collect previous and next span years\n",
    "    previous_span_years = get_span_years(partial_known, target_year, 'previous', prev_year_span)\n",
    "    next_span_years = get_span_years(partial_known, target_year, 'next', next_year_span)\n",
    "\n",
    "    # Step 2: Collect answers by year for both spans\n",
    "    previous_answers_by_year = collect_answers_by_year(partial_known, previous_span_years)\n",
    "    next_answers_by_year = collect_answers_by_year(partial_known, next_span_years)\n",
    "\n",
    "    # Initialize traversal steps log\n",
    "    traversal_steps = []\n",
    "\n",
    "    # Step 3: Check in previous span\n",
    "    for year in reversed(previous_span_years):  # Start from the latest previous year\n",
    "        traversal_steps.append('previous')\n",
    "        if tentative_ans.lower() in previous_answers_by_year.get(year, set()):\n",
    "            break  # Exit once found in previous span\n",
    "\n",
    "    # Step 4: If not found in previous, check in next span\n",
    "    # Only continue checking next span if tentative_ans was not found in previous span\n",
    "    if tentative_ans.lower() not in set(ans for sublist in previous_answers_by_year.values() for ans in sublist):\n",
    "        for year in next_span_years:\n",
    "            traversal_steps.append('next')\n",
    "            if tentative_ans.lower() in next_answers_by_year.get(year, set()):\n",
    "                break  # Exit once found in next span\n",
    "\n",
    "    # Step 5: Prepare traversal details\n",
    "    traversal_details = {\n",
    "        \"steps\": traversal_steps\n",
    "    }\n",
    "\n",
    "    return traversal_details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1-2. Get the traversal information for all domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_list = [\"Llama3.1_8B\", \"Mistral7B\", \"Phi3.5_Mini\", \"SOLAR_10.7B\", \"Gemma2_9B\", 'gpt-4o-mini']\n",
    "domain = \"General\" # General, Biomedical, Legal\n",
    "tentative_ans = \"Entity\"\n",
    "prev_span = 3\n",
    "next_span = 3\n",
    "\n",
    "for model_name in model_name_list:\n",
    "    bench_data_c, bench_data_u, timestamp_c, timestamp_u = load_data_with_updated_timestamp(model_name, domain)\n",
    "\n",
    "    partial_known_indices = [i for i, entry in enumerate(timestamp_c[\"Partial_known\"]) if entry]\n",
    "    subset_bench_data_c = [bench_data_c[i] for i in partial_known_indices]\n",
    "\n",
    "    for index, triplet in tqdm(zip(partial_known_indices, subset_bench_data_c), total=len(partial_known_indices), desc=\"Getting year spans...\"):\n",
    "        partial_known = timestamp_c[\"Partial_known\"][index][1]\n",
    "        for category in tqdm([\"incorrect\", \"partial_correct2\", \"partial_correct1\"], desc=\"Categories\", leave=False):\n",
    "            for year, objects_year in partial_known.items():\n",
    "                if objects_year[\"category\"] == category:\n",
    "                    chrono_ans = get_traversal_details(partial_known, year, tentative_ans, prev_year_span=prev_span, next_year_span=next_span)\n",
    "                    # Extract the 'steps' from traversal_steps\n",
    "                    steps = chrono_ans[\"steps\"]\n",
    "                    # Add 'steps' key to the current objects_year\n",
    "                    partial_known[year]['steps'] = steps\n",
    "\n",
    "        save_updated_timestamp(timestamp_c, f'./ChronoGap/{model_name}/Updated_Timestamp_{domain}_Dynamic_step.json')\n",
    "        \n",
    "        \n",
    "    partial_known_indices = [i for i, entry in enumerate(timestamp_u[\"Partial_known\"]) if entry]\n",
    "    subset_bench_data_u = [bench_data_u[i] for i in partial_known_indices]\n",
    "\n",
    "    for index, triplet in tqdm(zip(partial_known_indices, subset_bench_data_u), total=len(partial_known_indices), desc=\"Getting year spans...\"):\n",
    "        partial_known = timestamp_u[\"Partial_known\"][index][1]\n",
    "        for category in tqdm([\"incorrect\", \"partial_correct2\", \"partial_correct1\"], desc=\"Categories\", leave=False):\n",
    "            for year, objects_year in partial_known.items():\n",
    "                if objects_year[\"category\"] == category:\n",
    "                    chrono_ans = get_traversal_details(partial_known, year, tentative_ans, prev_year_span=prev_span, next_year_span=next_span)\n",
    "                    # Extract the 'steps' from traversal_steps\n",
    "                    steps = chrono_ans[\"steps\"]\n",
    "                    # Add 'steps' key to the current objects_year\n",
    "                    partial_known[year]['steps'] = steps\n",
    "\n",
    "        save_updated_timestamp(timestamp_u, f'./ChronoGap/{model_name}/Updated_Timestamp_{domain}_Static_step.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Results of Total Span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transition_rule_last_entry(entry, benchmark):\n",
    "    \"\"\"\n",
    "    Define the rule for moving an entry from 'Partial_known' to 'Known',\n",
    "    but only consider the last chrono_ans in each year for fuzzy matching.\n",
    "    \n",
    "    Returns:\n",
    "        - all_known: Whether all years are 'chrono_known' or 'correct'\n",
    "    \"\"\"\n",
    "    all_known = True\n",
    "\n",
    "    for year, year_data in entry.items():\n",
    "        if 'chrono_ans' in year_data:\n",
    "            # Clean chrono_ans by removing invalid entries\n",
    "            chrono_ans_list = clean_chrono_ans_list(year_data['chrono_ans'])\n",
    "            year_data['chrono_ans'] = chrono_ans_list\n",
    "\n",
    "            # Check only the last entry in the chrono_ans list for fuzzy matching\n",
    "            if chrono_ans_list:\n",
    "                last_ans = chrono_ans_list[-1]  # Get the last chrono_ans entry\n",
    "\n",
    "                # Perform fuzzy matching with the last chrono_ans entry\n",
    "                if is_fuzz_match(last_ans, benchmark.get(year, set())):                    \n",
    "                    year_data['category'] = 'chrono_known'\n",
    "        # Check if the updated category is not 'chrono_known' or 'correct'\n",
    "        if year_data['category'] not in ['chrono_known', 'correct']:\n",
    "            all_known = False\n",
    "    \n",
    "    return all_known\n",
    "\n",
    "\n",
    "def results_of_total_span(new_data, bench_entries, bench_name):\n",
    "    \"\"\"\n",
    "    Classify data entries into Known, Unknown, Cut-off, and Partial_known categories,\n",
    "    while calculating the increase in Known items. The classification is based only\n",
    "    on the last entry in the chrono_ans list.\n",
    "    \n",
    "    Returns:\n",
    "    - fine_grained_results: A dictionary with the counts of each category\n",
    "    - classification_indices: A list of items per category\n",
    "    - partial_known_to_known: The number of items moved from Partial_known to Known\n",
    "    - final_known_count: The final Known count after processing\n",
    "    \"\"\"\n",
    "    fine_grained_results = defaultdict(int)\n",
    "    classification_indices = defaultdict(list)\n",
    "    partial_known_to_known = 0 \n",
    "    moved_items_to_known = [] \n",
    "\n",
    "    for category in ['Known', 'Unknown', 'Cut-off', 'Partial_known']:\n",
    "        for idx, year_classifications in new_data.get(category, []):\n",
    "            benchmark = get_benchmark(bench_entries[idx], bench_name)\n",
    "            \n",
    "            if category == 'Partial_known':\n",
    "                all_known = transition_rule_last_entry(year_classifications, benchmark)\n",
    "                if all_known:\n",
    "                    final_category = 'Known'\n",
    "                    partial_known_to_known += 1\n",
    "                    moved_items_to_known.append((idx, year_classifications))\n",
    "    \n",
    "                else:\n",
    "                    final_category = 'Partial_known'\n",
    "            else:\n",
    "                final_category = category  \n",
    "\n",
    "            fine_grained_results[final_category] += 1\n",
    "            classification_indices[final_category].append((idx, year_classifications))\n",
    "\n",
    "    final_known_count = len(classification_indices['Known'])\n",
    "    previous_known_count = final_known_count - partial_known_to_known  \n",
    "\n",
    "    print(f\"Total items moved from Partial_known to Known: {partial_known_to_known}\")\n",
    "    \n",
    "    # Ensure no duplicate items during transition\n",
    "    for idx, year_classifications in moved_items_to_known:\n",
    "        classification_indices['Partial_known'] = [item for item in classification_indices['Partial_known'] if item[0] != idx]\n",
    "        if (idx, year_classifications) not in classification_indices['Known']:  # Prevent duplicates\n",
    "            classification_indices['Known'].append((idx, year_classifications))\n",
    "\n",
    "    # Debug final counts\n",
    "    print(f\"Final Known count: {len(classification_indices['Known'])}\")\n",
    "    print(f\"Final Partial_known count: {len(classification_indices['Partial_known'])}\")\n",
    "\n",
    "    # Ensure total count consistency\n",
    "    total_items = len(classification_indices['Known']) + len(classification_indices['Partial_known']) + \\\n",
    "                  len(classification_indices['Unknown']) + len(classification_indices['Cut-off'])\n",
    "    expected_total = len(new_data.get('Known', [])) + len(new_data.get('Partial_known', [])) + \\\n",
    "                     len(new_data.get('Unknown', [])) + len(new_data.get('Cut-off', []))\n",
    "\n",
    "    # Check if the total count is consistent\n",
    "    if total_items != expected_total:\n",
    "        raise ValueError(f\"Total item count mismatch! Expected: {expected_total}, Got: {total_items}\")\n",
    "\n",
    "    return fine_grained_results, classification_indices, previous_known_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_total_span(model_name_list, domain):\n",
    "    results_list_temp_state = []\n",
    "    \n",
    "    for t_state in [\"Dynamic\", \"Static\"]:\n",
    "        results_list = []\n",
    "\n",
    "        for model_name in tqdm(model_name_list):\n",
    "            bench, temp0_parsed_time, temp7_parsed_time = load_result(\n",
    "                                                                    model_name=model_name,\n",
    "                                                                    domain=domain,\n",
    "                                                                    temp_state=t_state,   \n",
    "                                                                    mode=\"generation\"\n",
    "                                                                    )\n",
    "            \n",
    "            new_data = read_json_file(f\"./ChronoGap/{model_name}/Updated_Timestamp_{domain}_{t_state}_step.json\")\n",
    "\n",
    "            results, indices, previous_known_count = results_of_total_span(\n",
    "                new_data, bench, domain\n",
    "            )\n",
    "\n",
    "            total = sum(results.values())\n",
    "\n",
    "            previous_known_percentage = (previous_known_count / total) * 100 if total > 0 else 0\n",
    "            current_known_count = results.get('Known', 0)\n",
    "            current_known_percentage = (current_known_count / total) * 100 if total > 0 else 0\n",
    "            known_increase_percentage = current_known_percentage - previous_known_percentage\n",
    "\n",
    "            print(f\"\\n[model: {model_name}, temp_state: {t_state}] ChroKnowPrompt results:\")\n",
    "            for category, count in results.items():\n",
    "                percentage = (count / total) * 100 if total > 0 else 0\n",
    "                if category == 'Known':\n",
    "                    print(f\"{category}: {count} ({percentage:.2f}%) (+{known_increase_percentage:.2f}% overall increase)\")\n",
    "                else:\n",
    "                    print(f\"{category}: {count} ({percentage:.2f}%)\")\n",
    "            print(f\"Total: {total}\")\n",
    "\n",
    "            results_list.append({\n",
    "                \"model_name\": model_name,\n",
    "                \"results\": results,\n",
    "                \"indices\": indices,\n",
    "                \"previous_known_count\": previous_known_count,\n",
    "                \"total\": total,\n",
    "                \"previous_known_percentage\": previous_known_percentage,\n",
    "                \"current_known_count\": current_known_count,\n",
    "                \"current_known_percentage\": current_known_percentage,\n",
    "                \"known_increase_percentage\": known_increase_percentage\n",
    "            })\n",
    "\n",
    "        results_list_temp_state.append(results_list)\n",
    "\n",
    "    return results_list_temp_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_list = [\"Llama3.1_8B\", \"Mistral7B\", \"Phi3.5_Mini\", \"SOLAR_10.7B\", \"Gemma2_9B\", 'gpt-4o-mini']\n",
    "\n",
    "# time variant domains: General, Biomedical, Legal\n",
    "domain = \"General\"\n",
    "\n",
    "results_list_temp_state = get_results_total_span(model_name_list=model_name_list,\n",
    "                 domain=domain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Results of Previous Span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transition_rule_previous_only_last_entry(entry, benchmark):\n",
    "    \"\"\"\n",
    "    Define the rule for moving an entry from 'Partial_known' to 'Known',\n",
    "    but only consider the last 'previous' step in chrono_ans for fuzzy matching.\n",
    "    \n",
    "    Returns:\n",
    "        - all_known: Whether all years are 'chrono_known' or 'correct'\n",
    "        - unknown_to_chrono_known_years: List of years where 'unknown' changed to 'chrono_known'\n",
    "    \"\"\"\n",
    "    all_known = True\n",
    "\n",
    "    for year, year_data in entry.items():\n",
    "        if 'chrono_ans' in year_data and 'steps' in year_data:\n",
    "            # Clean chrono_ans by removing invalid entries\n",
    "            chrono_ans_list = clean_chrono_ans_list(year_data['chrono_ans'])\n",
    "            steps_list = year_data['steps']\n",
    "            year_data['chrono_ans'] = chrono_ans_list\n",
    "\n",
    "            # Filter only the 'previous' steps\n",
    "            previous_steps = [(step, ans) for step, ans in zip(steps_list, chrono_ans_list) if step == \"previous\"]\n",
    "\n",
    "            # Only perform fuzzy matching on the last 'previous' step\n",
    "            if previous_steps:\n",
    "                last_previous_ans = previous_steps[-1][1]  # Get the last 'previous' step's chrono_ans\n",
    "\n",
    "                # Perform fuzzy matching with the last 'previous' chrono_ans entry\n",
    "                if is_fuzz_match(last_previous_ans, benchmark.get(year, set())):                    \n",
    "                    year_data['category'] = 'chrono_known'\n",
    "            # If no match or no 'previous' steps, retain the original category\n",
    "\n",
    "        # Check if the updated category is not 'chrono_known' or 'correct'\n",
    "        if year_data['category'] not in ['chrono_known', 'correct']:\n",
    "            all_known = False\n",
    "    \n",
    "    return all_known\n",
    "\n",
    "\n",
    "def results_of_previous_span(new_data, bench_entries, bench_name):\n",
    "    \"\"\"\n",
    "    Classify data entries into Known, Unknown, Cut-off, and Partial_known categories,\n",
    "    while considering only the last 'previous' step in chrono_ans for matching.\n",
    "    \n",
    "    Returns:\n",
    "    - fine_grained_results: A dictionary with the counts of each category\n",
    "    - classification_indices: A list of items per category\n",
    "    - partial_known_to_known: The number of items moved from Partial_known to Known\n",
    "    - final_known_count: The final Known count after processing\n",
    "    \"\"\"\n",
    "    fine_grained_results = defaultdict(int)\n",
    "    classification_indices = defaultdict(list)\n",
    "    partial_known_to_known = 0  \n",
    "    moved_items_to_known = []  \n",
    "\n",
    "    for category in ['Known', 'Unknown', 'Cut-off', 'Partial_known']:\n",
    "        for idx, year_classifications in new_data.get(category, []):\n",
    "            benchmark = get_benchmark(bench_entries[idx], bench_name)\n",
    "            \n",
    "            if category == 'Partial_known':\n",
    "                all_known  = transition_rule_previous_only_last_entry(year_classifications, benchmark)\n",
    "                if all_known:\n",
    "                    final_category = 'Known'\n",
    "                    partial_known_to_known += 1\n",
    "                    moved_items_to_known.append((idx, year_classifications))\n",
    "                \n",
    "                else:\n",
    "                    final_category = 'Partial_known'\n",
    "            else:\n",
    "                final_category = category\n",
    "\n",
    "            fine_grained_results[final_category] += 1\n",
    "            classification_indices[final_category].append((idx, year_classifications))\n",
    "\n",
    "\n",
    "    final_known_count = len(classification_indices['Known'])\n",
    "    previous_known_count = final_known_count - partial_known_to_known  \n",
    "\n",
    "    print(f\"Total items moved from Partial_known to Known: {partial_known_to_known}\")\n",
    "    \n",
    "    for idx, year_classifications in moved_items_to_known:\n",
    "        classification_indices['Partial_known'] = [item for item in classification_indices['Partial_known'] if item[0] != idx]\n",
    "        if (idx, year_classifications) not in classification_indices['Known']:  # Prevent duplicates\n",
    "            classification_indices['Known'].append((idx, year_classifications))\n",
    "\n",
    "    # Debug final counts\n",
    "    print(f\"Final Known count: {len(classification_indices['Known'])}\")\n",
    "    print(f\"Final Partial_known count: {len(classification_indices['Partial_known'])}\")\n",
    "\n",
    "    # Ensure total count consistency\n",
    "    total_items = len(classification_indices['Known']) + len(classification_indices['Partial_known']) + \\\n",
    "                  len(classification_indices['Unknown']) + len(classification_indices['Cut-off'])\n",
    "    expected_total = len(new_data.get('Known', [])) + len(new_data.get('Partial_known', [])) + \\\n",
    "                     len(new_data.get('Unknown', [])) + len(new_data.get('Cut-off', []))\n",
    "\n",
    "    # Check if the total count is consistent\n",
    "    if total_items != expected_total:\n",
    "        raise ValueError(f\"Total item count mismatch! Expected: {expected_total}, Got: {total_items}\")\n",
    "\n",
    "    return fine_grained_results, classification_indices, previous_known_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_previous_span(model_name_list, domain):\n",
    "    results_list_temp_state = []\n",
    "    \n",
    "    for t_state in [\"Dynamic\", \"Static\"]:\n",
    "        results_list = []\n",
    "\n",
    "        for model_name in tqdm(model_name_list):\n",
    "            bench, temp0_parsed_time, temp7_parsed_time = load_result(\n",
    "                                                                    model_name=model_name,\n",
    "                                                                    domain=domain,\n",
    "                                                                    temp_state=t_state,   \n",
    "                                                                    mode=\"generation\"\n",
    "                                                                    )\n",
    "            \n",
    "            new_data = read_json_file(f\"./ChronoGap/{model_name}/Updated_Timestamp_{domain}_{t_state}_step.json\")\n",
    "\n",
    "            results, indices, previous_known_count = results_of_previous_span(\n",
    "                new_data, bench, domain\n",
    "            )\n",
    "\n",
    "            total = sum(results.values())\n",
    "\n",
    "            previous_known_percentage = (previous_known_count / total) * 100 if total > 0 else 0\n",
    "            current_known_count = results.get('Known', 0)\n",
    "            current_known_percentage = (current_known_count / total) * 100 if total > 0 else 0\n",
    "            known_increase_percentage = current_known_percentage - previous_known_percentage\n",
    "\n",
    "            print(f\"\\n[model: {model_name}, temp_state: {t_state}] ChroKnowPrompt results:\")\n",
    "            for category, count in results.items():\n",
    "                percentage = (count / total) * 100 if total > 0 else 0\n",
    "                if category == 'Known':\n",
    "                    print(f\"{category}: {count} ({percentage:.2f}%) (+{known_increase_percentage:.2f}% overall increase)\")\n",
    "                else:\n",
    "                    print(f\"{category}: {count} ({percentage:.2f}%)\")\n",
    "            print(f\"Total: {total}\")\n",
    "\n",
    "            results_list.append({\n",
    "                \"model_name\": model_name,\n",
    "                \"results\": results,\n",
    "                \"indices\": indices,\n",
    "                \"previous_known_count\": previous_known_count,\n",
    "                \"total\": total,\n",
    "                \"previous_known_percentage\": previous_known_percentage,\n",
    "                \"current_known_count\": current_known_count,\n",
    "                \"current_known_percentage\": current_known_percentage,\n",
    "                \"known_increase_percentage\": known_increase_percentage\n",
    "            })\n",
    "\n",
    "        results_list_temp_state.append(results_list)\n",
    "\n",
    "    return results_list_temp_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_list = [\"Llama3.1_8B\", \"Mistral7B\", \"Phi3.5_Mini\", \"SOLAR_10.7B\", \"Gemma2_9B\", 'gpt-4o-mini']\n",
    "\n",
    "# time variant domains: General, Biomedical, Legal\n",
    "domain = \"General\"\n",
    "\n",
    "results_list_temp_state = get_results_previous_span(model_name_list=model_name_list,\n",
    "                 domain=domain)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ckcheck",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}