make_adv_case.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset, load_dataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import joblib, os, sys, time, random, json\n",
    "from openai import OpenAI\n",
    "from nltk import sent_tokenize\n",
    "import spacy\n",
    "from tqdm.auto import tqdm\n",
    "import joblib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25866\n",
      "25866\n",
      "25866\n"
     ]
    }
   ],
   "source": [
    "conflict = pd.DataFrame(load_dataset(\"Seongill/squad_conflict_v2_under_150_with_substitution\", split=\"train\"))\n",
    "conflict['answer_sent'] = conflict.apply(lambda x: find_answer_sent(x['context'], x['answer']), axis=1)\n",
    "print(len(conflict))\n",
    "conflict = conflict.dropna(subset=['answer_sent'])\n",
    "print(len(conflict))\n",
    "conflict[\"rewritten_answer_sent\"] = conflict.apply(lambda x: find_answer_sent(x['rewritten_context'], x['answer']), axis=1)\n",
    "conflict = conflict.dropna(subset=['rewritten_answer_sent'])\n",
    "print(len(conflict))\n",
    "conflict['answer_chunk'] = conflict.apply(lambda x: parser_answer_chunk(x['context'], x['answer']), axis=1)\n",
    "conflict['rewritten_answer_chunk'] = conflict.apply(lambda x: parser_answer_chunk(x['rewritten_context'], x['answer']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9eb6dfc1cbe34c82a0bd258204a5b942",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb4e0a8c8df84bee93a63075978428e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/26 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "Dataset.from_pandas(conflict.drop(columns=[\"title\",\"__index_level_0__\"], axis=1)).push_to_hub(\"Seongill/squad_conflict_v2_under_150_with_substitution_chunked\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'conflict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mconflict\u001b[49m\u001b[38;5;241m.\u001b[39mhead()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'conflict' is not defined"
     ]
    }
   ],
   "source": [
    "conflict.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answers</th>\n",
       "      <th>masked_query</th>\n",
       "      <th>query_embedding</th>\n",
       "      <th>ent_type</th>\n",
       "      <th>answer</th>\n",
       "      <th>random_answer</th>\n",
       "      <th>similar_answer</th>\n",
       "      <th>rewritten_context</th>\n",
       "      <th>has_answer</th>\n",
       "      <th>__index_level_0__</th>\n",
       "      <th>answer_sent</th>\n",
       "      <th>rewritten_answer_sent</th>\n",
       "      <th>rewritten_answer_chunk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5733be284776f41900661180</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "      <td>{'answer_start': [279], 'text': ['the Main Bui...</td>\n",
       "      <td>[MASK] at [MASK] is beside to which structure?</td>\n",
       "      <td>[0.3204971253871918, 0.20272424817085266, 0.22...</td>\n",
       "      <td>FAC</td>\n",
       "      <td>the Main Building</td>\n",
       "      <td>U.S. 1</td>\n",
       "      <td>the Royal Mews</td>\n",
       "      <td>With its Catholic character, the school's arch...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>Atop the Main Building's gold dome is a golden...</td>\n",
       "      <td>On top of the Main Building's dome, there is a...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5733bed24776f41900661188</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>The university is the major seat of the Congre...</td>\n",
       "      <td>Where is the headquarters of the Congregation ...</td>\n",
       "      <td>{'answer_start': [119], 'text': ['Rome']}</td>\n",
       "      <td>Where is the headquarters of [MASK]?</td>\n",
       "      <td>[0.3285101652145386, 0.22826211154460907, 0.12...</td>\n",
       "      <td>GPE</td>\n",
       "      <td>Rome</td>\n",
       "      <td>Baden-Württemberg</td>\n",
       "      <td>Antioch</td>\n",
       "      <td>Moreau Seminary, located on the campus borderi...</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>The university is the major seat of the Congre...</td>\n",
       "      <td>Although its official headquarters are in Rome...</td>\n",
       "      <td>The university is the major seat of the Congre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5733bed24776f41900661189</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>The university is the major seat of the Congre...</td>\n",
       "      <td>What is the primary seminary of the Congregati...</td>\n",
       "      <td>{'answer_start': [145], 'text': ['Moreau Semin...</td>\n",
       "      <td>What is the primary seminary of [MASK]?</td>\n",
       "      <td>[0.1649821400642395, 0.1607779562473297, 0.091...</td>\n",
       "      <td>ORG</td>\n",
       "      <td>Moreau Seminary</td>\n",
       "      <td>The Entertainment Channel</td>\n",
       "      <td>Hofstra University</td>\n",
       "      <td>Moreau Seminary, located on the campus borderi...</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>Its main seminary, Moreau Seminary, is located...</td>\n",
       "      <td>Moreau Seminary, located on the campus borderi...</td>\n",
       "      <td>Old College, the oldest building on campus and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5733bed24776f4190066118a</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>The university is the major seat of the Congre...</td>\n",
       "      <td>What is the oldest structure at Notre Dame?</td>\n",
       "      <td>{'answer_start': [234], 'text': ['Old College']}</td>\n",
       "      <td>What is the oldest structure at [MASK]?</td>\n",
       "      <td>[-0.02514331042766571, 0.3645743131637573, -0....</td>\n",
       "      <td>ORG</td>\n",
       "      <td>Old College</td>\n",
       "      <td>the Insurance Institute for Highway Safety</td>\n",
       "      <td>Eton College</td>\n",
       "      <td>Moreau Seminary, located on the campus borderi...</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>Old College, the oldest building on campus and...</td>\n",
       "      <td>Undergraduate seminarians reside in Old Colleg...</td>\n",
       "      <td>The university is the major seat of the Congre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5733a6424776f41900660f51</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>The College of Engineering was established in ...</td>\n",
       "      <td>How many BS level degrees are offered in the C...</td>\n",
       "      <td>{'answer_start': [487], 'text': ['eight']}</td>\n",
       "      <td>How many BS level degrees are offered in [MASK...</td>\n",
       "      <td>[0.6788813471794128, 0.5213160514831543, 0.146...</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>eight</td>\n",
       "      <td>2.1 million</td>\n",
       "      <td>over twenty</td>\n",
       "      <td>The College of Engineering was founded in 1920...</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "      <td>Today the college, housed in the Fitzpatrick, ...</td>\n",
       "      <td>There are eight B.S.</td>\n",
       "      <td>The College of Engineering was established in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25861</th>\n",
       "      <td>57378e311c456719005744b1</td>\n",
       "      <td>Force</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "      <td>How many scalar equations were formed into a s...</td>\n",
       "      <td>{'answer_start': [159, 159, 159, 159], 'text':...</td>\n",
       "      <td>How many scalar equations were formed into a s...</td>\n",
       "      <td>[0.3210879862308502, 0.09044459462165833, 0.41...</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>20</td>\n",
       "      <td>sextuple</td>\n",
       "      <td>5 to 7</td>\n",
       "      <td>James Clerk Maxwell was responsible for fully ...</td>\n",
       "      <td>True</td>\n",
       "      <td>26794</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "      <td>He was able to unify earlier theories into a s...</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25862</th>\n",
       "      <td>57378e311c456719005744b2</td>\n",
       "      <td>Force</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "      <td>How many vector equations did Heaviside and Gi...</td>\n",
       "      <td>{'answer_start': [215, 215, 215, 215], 'text':...</td>\n",
       "      <td>How many vector equations did [MASK] and Gibbs...</td>\n",
       "      <td>[-0.013793256133794785, -0.11560852080583572, ...</td>\n",
       "      <td>CARDINAL</td>\n",
       "      <td>4</td>\n",
       "      <td>approximately 119,000</td>\n",
       "      <td>5 to 7</td>\n",
       "      <td>James Clerk Maxwell was responsible for fully ...</td>\n",
       "      <td>True</td>\n",
       "      <td>26795</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "      <td>James Clerk Maxwell was responsible for fully ...</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25863</th>\n",
       "      <td>57378e311c456719005744b3</td>\n",
       "      <td>Force</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "      <td>Who discovered that magnetic and electric coul...</td>\n",
       "      <td>{'answer_start': [444, 88, 444, 444], 'text': ...</td>\n",
       "      <td>Who discovered that magnetic and electric coul...</td>\n",
       "      <td>[0.1881973296403885, -0.14856640994548798, -0....</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>Maxwell</td>\n",
       "      <td>Daniel Robbins</td>\n",
       "      <td>Thomas Edward Gordon</td>\n",
       "      <td>James Clerk Maxwell was responsible for fully ...</td>\n",
       "      <td>True</td>\n",
       "      <td>26796</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "      <td>James Clerk Maxwell was responsible for fully ...</td>\n",
       "      <td>The origin of electric and magnetic fields wou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25864</th>\n",
       "      <td>57379829c3c5551400e51f41</td>\n",
       "      <td>Force</td>\n",
       "      <td>The weak force is due to the exchange of the h...</td>\n",
       "      <td>At what temperature do weak and electromagneti...</td>\n",
       "      <td>{'answer_start': [514, 501, 528, 501], 'text':...</td>\n",
       "      <td>At what temperature do weak and electromagneti...</td>\n",
       "      <td>[0.15145105123519897, -0.07466116547584534, 0....</td>\n",
       "      <td>QUANTITY</td>\n",
       "      <td>approximately 1015 kelvins</td>\n",
       "      <td>50 m</td>\n",
       "      <td>267 kilometers</td>\n",
       "      <td>The heavy W and Z bosons are responsible for t...</td>\n",
       "      <td>True</td>\n",
       "      <td>26797</td>\n",
       "      <td>A consistent electroweak theory has also been ...</td>\n",
       "      <td>A coherent electroweak theory has also been es...</td>\n",
       "      <td>Its most familiar effect is beta decay (of neu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25865</th>\n",
       "      <td>5737a9afc3c5551400e51f64</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What is the law of thermodynamics associated w...</td>\n",
       "      <td>{'answer_start': [331, 331, 331, 331], 'text':...</td>\n",
       "      <td>What is the law of thermodynamics associated w...</td>\n",
       "      <td>[-0.15129241347312927, 0.08457052707672119, 0....</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>Second</td>\n",
       "      <td>68th</td>\n",
       "      <td>second</td>\n",
       "      <td>Detailed treatment with statistical mechanics ...</td>\n",
       "      <td>True</td>\n",
       "      <td>26798</td>\n",
       "      <td>According to the Second law of thermodynamics,...</td>\n",
       "      <td>According to the Second law of thermodynamics,...</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25866 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             id                     title  \\\n",
       "0      5733be284776f41900661180  University_of_Notre_Dame   \n",
       "1      5733bed24776f41900661188  University_of_Notre_Dame   \n",
       "2      5733bed24776f41900661189  University_of_Notre_Dame   \n",
       "3      5733bed24776f4190066118a  University_of_Notre_Dame   \n",
       "4      5733a6424776f41900660f51  University_of_Notre_Dame   \n",
       "...                         ...                       ...   \n",
       "25861  57378e311c456719005744b1                     Force   \n",
       "25862  57378e311c456719005744b2                     Force   \n",
       "25863  57378e311c456719005744b3                     Force   \n",
       "25864  57379829c3c5551400e51f41                     Force   \n",
       "25865  5737a9afc3c5551400e51f64                     Force   \n",
       "\n",
       "                                                 context  \\\n",
       "0      Architecturally, the school has a Catholic cha...   \n",
       "1      The university is the major seat of the Congre...   \n",
       "2      The university is the major seat of the Congre...   \n",
       "3      The university is the major seat of the Congre...   \n",
       "4      The College of Engineering was established in ...   \n",
       "...                                                  ...   \n",
       "25861  The origin of electric and magnetic fields wou...   \n",
       "25862  The origin of electric and magnetic fields wou...   \n",
       "25863  The origin of electric and magnetic fields wou...   \n",
       "25864  The weak force is due to the exchange of the h...   \n",
       "25865  The connection between macroscopic nonconserva...   \n",
       "\n",
       "                                                question  \\\n",
       "0      The Basilica of the Sacred heart at Notre Dame...   \n",
       "1      Where is the headquarters of the Congregation ...   \n",
       "2      What is the primary seminary of the Congregati...   \n",
       "3            What is the oldest structure at Notre Dame?   \n",
       "4      How many BS level degrees are offered in the C...   \n",
       "...                                                  ...   \n",
       "25861  How many scalar equations were formed into a s...   \n",
       "25862  How many vector equations did Heaviside and Gi...   \n",
       "25863  Who discovered that magnetic and electric coul...   \n",
       "25864  At what temperature do weak and electromagneti...   \n",
       "25865  What is the law of thermodynamics associated w...   \n",
       "\n",
       "                                                 answers  \\\n",
       "0      {'answer_start': [279], 'text': ['the Main Bui...   \n",
       "1              {'answer_start': [119], 'text': ['Rome']}   \n",
       "2      {'answer_start': [145], 'text': ['Moreau Semin...   \n",
       "3       {'answer_start': [234], 'text': ['Old College']}   \n",
       "4             {'answer_start': [487], 'text': ['eight']}   \n",
       "...                                                  ...   \n",
       "25861  {'answer_start': [159, 159, 159, 159], 'text':...   \n",
       "25862  {'answer_start': [215, 215, 215, 215], 'text':...   \n",
       "25863  {'answer_start': [444, 88, 444, 444], 'text': ...   \n",
       "25864  {'answer_start': [514, 501, 528, 501], 'text':...   \n",
       "25865  {'answer_start': [331, 331, 331, 331], 'text':...   \n",
       "\n",
       "                                            masked_query  \\\n",
       "0         [MASK] at [MASK] is beside to which structure?   \n",
       "1                   Where is the headquarters of [MASK]?   \n",
       "2                What is the primary seminary of [MASK]?   \n",
       "3                What is the oldest structure at [MASK]?   \n",
       "4      How many BS level degrees are offered in [MASK...   \n",
       "...                                                  ...   \n",
       "25861  How many scalar equations were formed into a s...   \n",
       "25862  How many vector equations did [MASK] and Gibbs...   \n",
       "25863  Who discovered that magnetic and electric coul...   \n",
       "25864  At what temperature do weak and electromagneti...   \n",
       "25865  What is the law of thermodynamics associated w...   \n",
       "\n",
       "                                         query_embedding  ent_type  \\\n",
       "0      [0.3204971253871918, 0.20272424817085266, 0.22...       FAC   \n",
       "1      [0.3285101652145386, 0.22826211154460907, 0.12...       GPE   \n",
       "2      [0.1649821400642395, 0.1607779562473297, 0.091...       ORG   \n",
       "3      [-0.02514331042766571, 0.3645743131637573, -0....       ORG   \n",
       "4      [0.6788813471794128, 0.5213160514831543, 0.146...  CARDINAL   \n",
       "...                                                  ...       ...   \n",
       "25861  [0.3210879862308502, 0.09044459462165833, 0.41...  CARDINAL   \n",
       "25862  [-0.013793256133794785, -0.11560852080583572, ...  CARDINAL   \n",
       "25863  [0.1881973296403885, -0.14856640994548798, -0....    PERSON   \n",
       "25864  [0.15145105123519897, -0.07466116547584534, 0....  QUANTITY   \n",
       "25865  [-0.15129241347312927, 0.08457052707672119, 0....   ORDINAL   \n",
       "\n",
       "                           answer                               random_answer  \\\n",
       "0               the Main Building                                      U.S. 1   \n",
       "1                            Rome                           Baden-Württemberg   \n",
       "2                 Moreau Seminary                   The Entertainment Channel   \n",
       "3                     Old College  the Insurance Institute for Highway Safety   \n",
       "4                           eight                                 2.1 million   \n",
       "...                           ...                                         ...   \n",
       "25861                          20                                    sextuple   \n",
       "25862                           4                       approximately 119,000   \n",
       "25863                     Maxwell                              Daniel Robbins   \n",
       "25864  approximately 1015 kelvins                                        50 m   \n",
       "25865                      Second                                        68th   \n",
       "\n",
       "             similar_answer  \\\n",
       "0            the Royal Mews   \n",
       "1                   Antioch   \n",
       "2        Hofstra University   \n",
       "3              Eton College   \n",
       "4               over twenty   \n",
       "...                     ...   \n",
       "25861                5 to 7   \n",
       "25862                5 to 7   \n",
       "25863  Thomas Edward Gordon   \n",
       "25864        267 kilometers   \n",
       "25865                second   \n",
       "\n",
       "                                       rewritten_context  has_answer  \\\n",
       "0      With its Catholic character, the school's arch...        True   \n",
       "1      Moreau Seminary, located on the campus borderi...        True   \n",
       "2      Moreau Seminary, located on the campus borderi...        True   \n",
       "3      Moreau Seminary, located on the campus borderi...        True   \n",
       "4      The College of Engineering was founded in 1920...        True   \n",
       "...                                                  ...         ...   \n",
       "25861  James Clerk Maxwell was responsible for fully ...        True   \n",
       "25862  James Clerk Maxwell was responsible for fully ...        True   \n",
       "25863  James Clerk Maxwell was responsible for fully ...        True   \n",
       "25864  The heavy W and Z bosons are responsible for t...        True   \n",
       "25865  Detailed treatment with statistical mechanics ...        True   \n",
       "\n",
       "       __index_level_0__                                        answer_sent  \\\n",
       "0                      0  Atop the Main Building's gold dome is a golden...   \n",
       "1                      1  The university is the major seat of the Congre...   \n",
       "2                      2  Its main seminary, Moreau Seminary, is located...   \n",
       "3                      3  Old College, the oldest building on campus and...   \n",
       "4                      4  Today the college, housed in the Fitzpatrick, ...   \n",
       "...                  ...                                                ...   \n",
       "25861              26794  The origin of electric and magnetic fields wou...   \n",
       "25862              26795  The origin of electric and magnetic fields wou...   \n",
       "25863              26796  The origin of electric and magnetic fields wou...   \n",
       "25864              26797  A consistent electroweak theory has also been ...   \n",
       "25865              26798  According to the Second law of thermodynamics,...   \n",
       "\n",
       "                                   rewritten_answer_sent  \\\n",
       "0      On top of the Main Building's dome, there is a...   \n",
       "1      Although its official headquarters are in Rome...   \n",
       "2      Moreau Seminary, located on the campus borderi...   \n",
       "3      Undergraduate seminarians reside in Old Colleg...   \n",
       "4                                   There are eight B.S.   \n",
       "...                                                  ...   \n",
       "25861  He was able to unify earlier theories into a s...   \n",
       "25862  James Clerk Maxwell was responsible for fully ...   \n",
       "25863  James Clerk Maxwell was responsible for fully ...   \n",
       "25864  A coherent electroweak theory has also been es...   \n",
       "25865  According to the Second law of thermodynamics,...   \n",
       "\n",
       "                                  rewritten_answer_chunk  \n",
       "0      Architecturally, the school has a Catholic cha...  \n",
       "1      The university is the major seat of the Congre...  \n",
       "2      Old College, the oldest building on campus and...  \n",
       "3      The university is the major seat of the Congre...  \n",
       "4      The College of Engineering was established in ...  \n",
       "...                                                  ...  \n",
       "25861  The origin of electric and magnetic fields wou...  \n",
       "25862  The origin of electric and magnetic fields wou...  \n",
       "25863  The origin of electric and magnetic fields wou...  \n",
       "25864  Its most familiar effect is beta decay (of neu...  \n",
       "25865  The connection between macroscopic nonconserva...  \n",
       "\n",
       "[25866 rows x 17 columns]"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conflict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answers</th>\n",
       "      <th>masked_query</th>\n",
       "      <th>query_embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5733be284776f41900661182</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>{'answer_start': [515], 'text': ['Saint Bernad...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>[-0.04606574401259422, 0.015957484021782875, -...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id                     title  \\\n",
       "0  5733be284776f41900661182  University_of_Notre_Dame   \n",
       "\n",
       "                                             context  \\\n",
       "0  Architecturally, the school has a Catholic cha...   \n",
       "\n",
       "                                            question  \\\n",
       "0  To whom did the Virgin Mary allegedly appear i...   \n",
       "\n",
       "                                             answers  \\\n",
       "0  {'answer_start': [515], 'text': ['Saint Bernad...   \n",
       "\n",
       "                                        masked_query  \\\n",
       "0  To whom did the Virgin Mary allegedly appear i...   \n",
       "\n",
       "                                     query_embedding  \n",
       "0  [-0.04606574401259422, 0.015957484021782875, -...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "squad = pd.DataFrame(load_dataset(\"Seongill/SQuAD_unique_questions\", split=\"train\"))\n",
    "squad.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(97888, 8)\n",
      "(92286, 8)\n"
     ]
    }
   ],
   "source": [
    "df = squad.copy()\n",
    "df['answer_cnt'] = df['answers'].apply(lambda x: len(list(set(x['text']))))\n",
    "print(df.shape)\n",
    "df = df[df.answer_cnt == 1]\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_answer_sent(context, answer):\n",
    "    sents = sent_tokenize(context)\n",
    "    for sent in sents:\n",
    "        if answer in sent:\n",
    "            return sent\n",
    "    return None\n",
    "\n",
    "def parser_answer_chunk(context, answer):\n",
    "    sents = sent_tokenize(context)\n",
    "    for idx, sent in enumerate(sents):\n",
    "        if answer in sent:\n",
    "            answer_sent_idx = idx\n",
    "    start_idx = max(0, answer_sent_idx-3)\n",
    "    end_idx = min(len(sents), answer_sent_idx+4)\n",
    "    return ' '.join(sents[start_idx:end_idx])\n",
    "# df['answer_sent'] = df.apply(lambda x: find_answer_sent(x['context'], x['answers']['text'][0]), axis=1)\n",
    "# print(len(df))\n",
    "# df = df.dropna(subset=['answer_sent'])\n",
    "# print(len(df))\n",
    "# df['answer_chunk'] = df.apply(lambda x: parser_answer_chunk(x['context'], x['answers']['text'][0]), axis=1)\n",
    "# df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"answer\"] = df[\"answers\"].apply(lambda x: x[\"text\"][0])\n",
    "df = df.drop(columns=[\"answers\", \"answer_cnt\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(columns=[\"query_embedding\", \"masked_query\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answer_sent</th>\n",
       "      <th>answer_chunk</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5733be284776f41900661182</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>It is a replica of the grotto at Lourdes, Fran...</td>\n",
       "      <td>Atop the Main Building's gold dome is a golden...</td>\n",
       "      <td>Saint Bernadette Soubirous</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5733be284776f4190066117f</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is in front of the Notre Dame Main Building?</td>\n",
       "      <td>Immediately in front of the Main Building and ...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>a copper statue of Christ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5733be284776f41900661180</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "      <td>Atop the Main Building's gold dome is a golden...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>the Main Building</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5733be284776f41900661181</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is the Grotto at Notre Dame?</td>\n",
       "      <td>Immediately behind the basilica is the Grotto,...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>a Marian place of prayer and reflection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5733be284776f4190066117e</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What sits on top of the Main Building at Notre...</td>\n",
       "      <td>Atop the Main Building's gold dome is a golden...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>a golden statue of the Virgin Mary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97880</th>\n",
       "      <td>5737a9afc3c5551400e51f63</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What is the exchange of heat associated with?</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>nonconservative forces</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97882</th>\n",
       "      <td>5737a9afc3c5551400e51f65</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What makes energy changes in a closed system?</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>nonconservative forces</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97884</th>\n",
       "      <td>5737aafd1c456719005744fc</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What is the kilogram-force sometimes reffered ...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>kilopond</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97886</th>\n",
       "      <td>5737aafd1c456719005744fe</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What seldom used term of a unit of force equal...</td>\n",
       "      <td>Other arcane units of force include the sthène...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>kip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97887</th>\n",
       "      <td>5737aafd1c456719005744ff</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What is the seldom used force unit equal to on...</td>\n",
       "      <td>Other arcane units of force include the sthène...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>sthène</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92064 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             id                     title  \\\n",
       "0      5733be284776f41900661182  University_of_Notre_Dame   \n",
       "1      5733be284776f4190066117f  University_of_Notre_Dame   \n",
       "2      5733be284776f41900661180  University_of_Notre_Dame   \n",
       "3      5733be284776f41900661181  University_of_Notre_Dame   \n",
       "4      5733be284776f4190066117e  University_of_Notre_Dame   \n",
       "...                         ...                       ...   \n",
       "97880  5737a9afc3c5551400e51f63                     Force   \n",
       "97882  5737a9afc3c5551400e51f65                     Force   \n",
       "97884  5737aafd1c456719005744fc                     Force   \n",
       "97886  5737aafd1c456719005744fe                     Force   \n",
       "97887  5737aafd1c456719005744ff                     Force   \n",
       "\n",
       "                                                 context  \\\n",
       "0      Architecturally, the school has a Catholic cha...   \n",
       "1      Architecturally, the school has a Catholic cha...   \n",
       "2      Architecturally, the school has a Catholic cha...   \n",
       "3      Architecturally, the school has a Catholic cha...   \n",
       "4      Architecturally, the school has a Catholic cha...   \n",
       "...                                                  ...   \n",
       "97880  The connection between macroscopic nonconserva...   \n",
       "97882  The connection between macroscopic nonconserva...   \n",
       "97884  The pound-force has a metric counterpart, less...   \n",
       "97886  The pound-force has a metric counterpart, less...   \n",
       "97887  The pound-force has a metric counterpart, less...   \n",
       "\n",
       "                                                question  \\\n",
       "0      To whom did the Virgin Mary allegedly appear i...   \n",
       "1      What is in front of the Notre Dame Main Building?   \n",
       "2      The Basilica of the Sacred heart at Notre Dame...   \n",
       "3                      What is the Grotto at Notre Dame?   \n",
       "4      What sits on top of the Main Building at Notre...   \n",
       "...                                                  ...   \n",
       "97880      What is the exchange of heat associated with?   \n",
       "97882      What makes energy changes in a closed system?   \n",
       "97884  What is the kilogram-force sometimes reffered ...   \n",
       "97886  What seldom used term of a unit of force equal...   \n",
       "97887  What is the seldom used force unit equal to on...   \n",
       "\n",
       "                                             answer_sent  \\\n",
       "0      It is a replica of the grotto at Lourdes, Fran...   \n",
       "1      Immediately in front of the Main Building and ...   \n",
       "2      Atop the Main Building's gold dome is a golden...   \n",
       "3      Immediately behind the basilica is the Grotto,...   \n",
       "4      Atop the Main Building's gold dome is a golden...   \n",
       "...                                                  ...   \n",
       "97880  The connection between macroscopic nonconserva...   \n",
       "97882  The connection between macroscopic nonconserva...   \n",
       "97884  The pound-force has a metric counterpart, less...   \n",
       "97886  Other arcane units of force include the sthène...   \n",
       "97887  Other arcane units of force include the sthène...   \n",
       "\n",
       "                                            answer_chunk  \\\n",
       "0      Atop the Main Building's gold dome is a golden...   \n",
       "1      Architecturally, the school has a Catholic cha...   \n",
       "2      Architecturally, the school has a Catholic cha...   \n",
       "3      Architecturally, the school has a Catholic cha...   \n",
       "4      Architecturally, the school has a Catholic cha...   \n",
       "...                                                  ...   \n",
       "97880  The connection between macroscopic nonconserva...   \n",
       "97882  The connection between macroscopic nonconserva...   \n",
       "97884  The pound-force has a metric counterpart, less...   \n",
       "97886  The pound-force has a metric counterpart, less...   \n",
       "97887  The pound-force has a metric counterpart, less...   \n",
       "\n",
       "                                        answer  \n",
       "0                   Saint Bernadette Soubirous  \n",
       "1                    a copper statue of Christ  \n",
       "2                            the Main Building  \n",
       "3      a Marian place of prayer and reflection  \n",
       "4           a golden statue of the Virgin Mary  \n",
       "...                                        ...  \n",
       "97880                   nonconservative forces  \n",
       "97882                   nonconservative forces  \n",
       "97884                                 kilopond  \n",
       "97886                                      kip  \n",
       "97887                                   sthène  \n",
       "\n",
       "[92064 rows x 7 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.to_csv(\"squad.csv\", index=False)\n",
    "df = pd.read_csv(\"squad.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spacy.prefer_gpu()\n",
    "nlp = spacy.load(\"en_core_web_trf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe5a925a5ce64ab8a6d8bde8adf32dce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "total_ctxs = list(set(squad.context.tolist()))\n",
    "BATCH_SIZE = 2000\n",
    "docs = []\n",
    "for i in tqdm(range(0, len(total_ctxs), BATCH_SIZE)):\n",
    "    batch = total_ctxs[i:i+BATCH_SIZE]\n",
    "    docs.extend(list(nlp.pipe(batch)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95515 50872 18 6\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "ANSWER_POS = [\"ADV\", \"ADJ\", \"NOUN\", \"NUM\", \"SYM\", \"PROPN\"]\n",
    "ent2text = defaultdict(list)\n",
    "text2ent = dict()\n",
    "pos2text = defaultdict(list)\n",
    "text2pos = dict()\n",
    "for doc in docs:\n",
    "    for ent in doc.ents:\n",
    "        ent2text[ent.label_].append(ent.text)\n",
    "    for token in doc:\n",
    "        if not token.ent_type_:\n",
    "            pos2text[token.pos_].append(token.text)\n",
    "for k, v in ent2text.items():\n",
    "    ent2text[k] = list(set(v))\n",
    "for k, v in ent2text.items():\n",
    "    for vv in v:\n",
    "        text2ent[vv] = k\n",
    "pos2text = {k:list(set(v)) for k, v in pos2text.items() if k in ANSWER_POS}\n",
    "for k, v in pos2text.items():\n",
    "    for vv in v:\n",
    "        text2pos[vv] = k\n",
    "print(len(text2ent), len(text2pos), len(ent2text), len(pos2text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='answer_len'>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGxCAYAAACA4KdFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBkElEQVR4nO3de3iU9Z3//9fMJDNJSCYhQBIC4WAREDlYQWLWc6VEFv2JWlfRKkW0aze4Qqoivyraw/5icb1Wq6jb+vuK3V0r0i1WQaFshFAloMZGASWrFgwKSTiYmSTkOPP5/hHmhimnhBzumcnzcV1zZWbuz9zznrnlmpf353A7jDFGAAAAMcZpdwEAAAA9gZADAABiEiEHAADEJEIOAACISYQcAAAQkwg5AAAgJhFyAABATCLkAACAmBRndwF2CgaD2rt3r1JSUuRwOOwuBwAAdIAxRnV1dcrOzpbTefLzNX065Ozdu1c5OTl2lwEAAM7Anj17NHTo0JNu79MhJyUlRVL7l+T1em2uBgAAdITf71dOTo71O34yfTrkhLqovF4vIQcAgChzuqEmDDwGAAAxiZADAABiEiEHAADEJEIOAACISYQcAAAQkwg5AAAgJhFyAABATCLkAACAmETIAQAAMYmQAwAAYhIhBwAAxCRCDgAAiEmEnG5mjNH2r336044qNbUG7C4HAIA+i5DTzRwOh2b/eot++B9l+uqbw3aXAwBAn0XI6QHZaYmSpL21TTZXAgBA30XI6QGD0xIkSXtrG22uBACAvouQ0wOsMzk+zuQAAGAXQk4PGGJ1V3EmBwAAuxByesDgVLqrAACwGyGnB4S6q/bRXQUAgG0IOT0g1F31dW2jjDE2VwMAQN9EyOkBmd4EORxSS1tQBxta7C4HAIA+iZDTA9xxTg1K9kiS9rFWDgAAtiDk9JDsY7qsAABA7yPk9JBsFgQEAMBWhJwekp0ammFFyAEAwA6EnB7C9asAALAXIaeHhLqrGJMDAIA9CDk95OiCgIQcAADsQMjpIaGQU1PXrJa2oM3VAADQ9xByesiAfm6545wyRqr2My4HAIDeRsjpIQ6HQ9lcqBMAANsQcnqQNcOKcTkAAPQ6Qk4PGpzKNHIAAOxCyOlBQ1j1GAAA23Qq5Dz33HOaOHGivF6vvF6v8vLy9NZbb1nbm5qaVFBQoAEDBig5OVk33HCDqqurw/ZRWVmpmTNnKikpSRkZGbr//vvV1tYW1mbjxo06//zz5fF4NGrUKC1fvvy4WpYtW6YRI0YoISFBubm5eu+99zrzUXrF0QUBCTkAAPS2ToWcoUOH6rHHHlNZWZk++OADfec739G1116rHTt2SJIWLlyoN954QytXrlRJSYn27t2r66+/3np9IBDQzJkz1dLSos2bN+ull17S8uXLtWTJEqvNrl27NHPmTF1xxRUqLy/XggULdOedd2rdunVWmxUrVqiwsFCPPPKIPvzwQ02aNEn5+fmqqanp6vfRrQaz6jEAAPYxXdS/f3/zwgsvmNraWhMfH29Wrlxpbfv000+NJFNaWmqMMebNN980TqfTVFVVWW2ee+454/V6TXNzszHGmAceeMCce+65Ye9x0003mfz8fOvx1KlTTUFBgfU4EAiY7OxsU1RU1KnafT6fkWR8Pl+nXtdRn1X7zfBFq834R9b2yP4BAOiLOvr7fcZjcgKBgF555RU1NDQoLy9PZWVlam1t1bRp06w2Y8eO1bBhw1RaWipJKi0t1YQJE5SZmWm1yc/Pl9/vt84GlZaWhu0j1Ca0j5aWFpWVlYW1cTqdmjZtmtUmUoQGHtc1tamuqdXmagAA6FviOvuCbdu2KS8vT01NTUpOTtaqVas0btw4lZeXy+12Ky0tLax9ZmamqqqqJElVVVVhASe0PbTtVG38fr8aGxv1zTffKBAInLDNzp07T1l7c3Ozmpubrcd+v7/jH/wM9PPEKTUxXr7GVu3zNSklIb5H3w8AABzV6TM5Y8aMUXl5ubZu3aof/ehHmjNnjj755JOeqK3bFRUVKTU11brl5OT0+HuGBh9zoU4AAHpXp0OO2+3WqFGjNHnyZBUVFWnSpEl66qmnlJWVpZaWFtXW1oa1r66uVlZWliQpKyvruNlWocena+P1epWYmKiBAwfK5XKdsE1oHyezePFi+Xw+67Znz57OfvxOY9VjAADs0eV1coLBoJqbmzV58mTFx8eruLjY2lZRUaHKykrl5eVJkvLy8rRt27awWVDr16+X1+vVuHHjrDbH7iPUJrQPt9utyZMnh7UJBoMqLi622pyMx+Oxpr+Hbj3Nuho5M6wAAOhVnRqTs3jxYs2YMUPDhg1TXV2dXn75ZW3cuFHr1q1Tamqq5s2bp8LCQqWnp8vr9eqee+5RXl6eLrzwQknS9OnTNW7cON12221aunSpqqqq9NBDD6mgoEAej0eSdPfdd+uZZ57RAw88oDvuuENvv/22Xn31Va1Zs8aqo7CwUHPmzNGUKVM0depUPfnkk2poaNDcuXO78avpHqyVAwCAPToVcmpqanT77bdr3759Sk1N1cSJE7Vu3Tp997vflST927/9m5xOp2644QY1NzcrPz9fzz77rPV6l8ul1atX60c/+pHy8vLUr18/zZkzRz/72c+sNiNHjtSaNWu0cOFCPfXUUxo6dKheeOEF5efnW21uuukm7d+/X0uWLFFVVZXOO+88rV279rjByJEg+8iqx4zJAQCgdzmMMcbuIuzi9/uVmpoqn8/XY11X7+8+pBufL9Ww9CRteuCKHnkPAAD6ko7+fnPtqh5mjcnxNSoY7LN5EgCAXkfI6WGZKR45HVJrwOhAffPpXwAAALoFIaeHxbmcyvQemUbuY4YVAAC9hZDTC5hhBQBA7yPk9ILBLAgIAECvI+T0giHWmRy6qwAA6C2EnF5AdxUAAL2PkNMLrO4qHyEHAIDeQsjpBdl0VwEA0OsIOb0gNCbnQH2zmloDNlcDAEDfQMjpBWlJ8UqIb/+qq1grBwCAXkHI6QUOh+NolxXjcgAA6BWEnF7CNHIAAHoXIaeXsCAgAAC9i5DTS469GjkAAOh5hJxeEgo5X9NdBQBAryDk9JLs1CNncuiuAgCgVxByekl22tExOcYYm6sBACD2EXJ6yeAjZ3IaWgLyN7bZXA0AALGPkNNLEt0upfdzS2KtHAAAegMhpxcd22UFAAB6FiGnF4W6rAg5AAD0PEJOL7JWPeb6VQAA9DhCTi+iuwoAgN5DyOlFdFcBANB7CDm9KJuLdAIA0GsIOb0oNCanyt+kQJAFAQEA6EmEnF40KMWjOKdDgaBRTR1ncwAA6EmEnF7kcjqU6Q0NPibkAADQkwg5vcyaRs7gYwAAehQhp5cNZho5AAC9gpDTy0IzrPaxICAAAD2KkNPLQiHna87kAADQowg5vSw7le4qAAB6AyGnl9FdBQBA7yDk9LJQyDnU0KLGloDN1QAAELsIOb3MmxCnfm6XJGmvjy4rAAB6CiGnlzkcjqNdViwICABAjyHk2CCbBQEBAOhxhBwbZIcWBKS7CgCAHkPIsUF2KmdyAADoaZ0KOUVFRbrggguUkpKijIwMzZo1SxUVFWFtLr/8cjkcjrDb3XffHdamsrJSM2fOVFJSkjIyMnT//ferra0trM3GjRt1/vnny+PxaNSoUVq+fPlx9SxbtkwjRoxQQkKCcnNz9d5773Xm49jmaHcVY3IAAOgpnQo5JSUlKigo0JYtW7R+/Xq1trZq+vTpamhoCGt31113ad++fdZt6dKl1rZAIKCZM2eqpaVFmzdv1ksvvaTly5dryZIlVptdu3Zp5syZuuKKK1ReXq4FCxbozjvv1Lp166w2K1asUGFhoR555BF9+OGHmjRpkvLz81VTU3Om30WvGUx3FQAAPc5hjDFn+uL9+/crIyNDJSUluvTSSyW1n8k577zz9OSTT57wNW+99Zauvvpq7d27V5mZmZKk559/XosWLdL+/fvldru1aNEirVmzRtu3b7ded/PNN6u2tlZr166VJOXm5uqCCy7QM888I0kKBoPKycnRPffcowcffLBD9fv9fqWmpsrn88nr9Z7p19BpXx5s0GWPb1RCvFOf/uwqORyOXntvAACiXUd/v7s0Jsfn80mS0tPTw57/r//6Lw0cOFDjx4/X4sWLdfjwYWtbaWmpJkyYYAUcScrPz5ff79eOHTusNtOmTQvbZ35+vkpLSyVJLS0tKisrC2vjdDo1bdo0q82JNDc3y+/3h93skHXk0g5NrUF9c7jVlhoAAIh1cWf6wmAwqAULFuiiiy7S+PHjredvueUWDR8+XNnZ2fr444+1aNEiVVRU6A9/+IMkqaqqKizgSLIeV1VVnbKN3+9XY2OjvvnmGwUCgRO22blz50lrLioq0k9/+tMz/cjdxhPn0sBkjw7UN2tvbaPS+7ntLgkAgJhzxiGnoKBA27dv1zvvvBP2/A9/+EPr/oQJEzR48GBdeeWV+uKLL/Stb33rzCvtBosXL1ZhYaH12O/3Kycnx5ZahqQlWCFn/JBUW2oAACCWnVF31fz587V69Wpt2LBBQ4cOPWXb3NxcSdLnn38uScrKylJ1dXVYm9DjrKysU7bxer1KTEzUwIED5XK5TtgmtI8T8Xg88nq9YTe7DGYaOQAAPapTIccYo/nz52vVqlV6++23NXLkyNO+pry8XJI0ePBgSVJeXp62bdsWNgtq/fr18nq9GjdunNWmuLg4bD/r169XXl6eJMntdmvy5MlhbYLBoIqLi602kY6rkQMA0LM61V1VUFCgl19+WX/84x+VkpJijaFJTU1VYmKivvjiC7388sv6+7//ew0YMEAff/yxFi5cqEsvvVQTJ06UJE2fPl3jxo3TbbfdpqVLl6qqqkoPPfSQCgoK5PF4JEl33323nnnmGT3wwAO644479Pbbb+vVV1/VmjVrrFoKCws1Z84cTZkyRVOnTtWTTz6phoYGzZ07t7u+mx4VWvX4a87kAADQM0wnSDrh7cUXXzTGGFNZWWkuvfRSk56ebjwejxk1apS5//77jc/nC9vP7t27zYwZM0xiYqIZOHCg+fGPf2xaW1vD2mzYsMGcd955xu12m7POOst6j2M9/fTTZtiwYcbtdpupU6eaLVu2dObjGJ/PZyQdV19vWPPxXjN80Wpz3bJ3ev29AQCIZh39/e7SOjnRzq51ciSpfE+tZi17V4NTE1S6+MpefW8AAKJZr6yTgzMX6q6q9jepNRC0uRoAAGIPIccmA/t5FO9yKGjagw4AAOhehBybOJ0Oaxo5M6wAAOh+hBwbhbqsWCsHAIDuR8ixUfaRMzlMIwcAoPsRcmxkLQhYS3cVAADdjZBjo1DIobsKAIDuR8ix0WBWPQYAoMcQcmw0hOtXAQDQYwg5Nsr0tp/J8TW2qqk1YHM1AADEFkKOjbwJcUqMd0mSqjibAwBAtyLk2MjhcCgrtf1sThWrHgMA0K0IOTbL9HokcSYHAIDuRsixWejSDpzJAQCgexFybBYafMyZHAAAuhchx2ZZR7qruBI5AADdi5BjMwYeAwDQMwg5Ngt1V1XTXQUAQLci5NgsdCanpq5ZgaCxuRoAAGIHIcdmg5I9cjqktqDRwfpmu8sBACBmEHJsFudyalDKkbVyGJcDAEC3IeREgCymkQMA0O0IORHAGnzMmRwAALoNIScCMI0cAIDuR8iJAEdXPWbgMQAA3YWQEwGy6K4CAKDbEXIiAN1VAAB0P0JOBOAinQAAdD9CTgQIncmpb25TfXObzdUAABAbCDkRINkTpxRPnCTO5gAA0F0IOREiM5XBxwAAdCdCToRg1WMAALoXISdCWIOPOZMDAEC3IOREiKzU9ot00l0FAED3IORECLqrAADoXoScCJGVmiiJ7ioAALoLISdCcCYHAIDuRciJEJlHxuQcqG9WWyBoczUAAEQ/Qk6EGNjPozinQ0Ej7a/nauQAAHQVISdCOJ0OZaS0n82hywoAgK4j5EQQVj0GAKD7dCrkFBUV6YILLlBKSooyMjI0a9YsVVRUhLVpampSQUGBBgwYoOTkZN1www2qrq4Oa1NZWamZM2cqKSlJGRkZuv/++9XWFn5hyo0bN+r888+Xx+PRqFGjtHz58uPqWbZsmUaMGKGEhATl5ubqvffe68zHiTgMPgYAoPt0KuSUlJSooKBAW7Zs0fr169Xa2qrp06eroaHBarNw4UK98cYbWrlypUpKSrR3715df/311vZAIKCZM2eqpaVFmzdv1ksvvaTly5dryZIlVptdu3Zp5syZuuKKK1ReXq4FCxbozjvv1Lp166w2K1asUGFhoR555BF9+OGHmjRpkvLz81VTU9OV78NWR1c9ZkwOAABdZrqgpqbGSDIlJSXGGGNqa2tNfHy8WblypdXm008/NZJMaWmpMcaYN9980zidTlNVVWW1ee6554zX6zXNzc3GGGMeeOABc+6554a910033WTy8/Otx1OnTjUFBQXW40AgYLKzs01RUVGH6/f5fEaS8fl8nfjUPef5jZ+b4YtWm3t/96HdpQAAELE6+vvdpTE5Pp9PkpSeni5JKisrU2trq6ZNm2a1GTt2rIYNG6bS0lJJUmlpqSZMmKDMzEyrTX5+vvx+v3bs2GG1OXYfoTahfbS0tKisrCysjdPp1LRp06w2J9Lc3Cy/3x92iyRZqVy/CgCA7nLGIScYDGrBggW66KKLNH78eElSVVWV3G630tLSwtpmZmaqqqrKanNswAltD207VRu/36/GxkYdOHBAgUDghG1C+ziRoqIipaamWrecnJzOf/AeFOquqqa7CgCALjvjkFNQUKDt27frlVde6c56etTixYvl8/ms2549e+wuKcyxA4+NMTZXAwBAdIs7kxfNnz9fq1ev1qZNmzR06FDr+aysLLW0tKi2tjbsbE51dbWysrKsNn87Cyo0++rYNn87I6u6ulper1eJiYlyuVxyuVwnbBPax4l4PB55PJ7Of+BeEuquamwNyN/UptTEeJsrAgAgenXqTI4xRvPnz9eqVav09ttva+TIkWHbJ0+erPj4eBUXF1vPVVRUqLKyUnl5eZKkvLw8bdu2LWwW1Pr16+X1ejVu3DirzbH7CLUJ7cPtdmvy5MlhbYLBoIqLi6020Sgh3mUFG9bKAQCgazp1JqegoEAvv/yy/vjHPyolJcUa/5KamqrExESlpqZq3rx5KiwsVHp6urxer+655x7l5eXpwgsvlCRNnz5d48aN02233aalS5eqqqpKDz30kAoKCqyzLHfffbeeeeYZPfDAA7rjjjv09ttv69VXX9WaNWusWgoLCzVnzhxNmTJFU6dO1ZNPPqmGhgbNnTu3u74bW2R5E+RrbFWVr0mjM1PsLgcAgOjVmSlbkk54e/HFF602jY2N5p/+6Z9M//79TVJSkrnuuuvMvn37wvaze/duM2PGDJOYmGgGDhxofvzjH5vW1tawNhs2bDDnnXeecbvd5qyzzgp7j5Cnn37aDBs2zLjdbjN16lSzZcuWznyciJtCbowxt///W83wRavNivcq7S4FAICI1NHfb4cxfXeEq9/vV2pqqnw+n7xer93lSJIW/f5jrfhgjwq/O1r/fOXZdpcDAEDE6ejvN9euijCZrJUDAEC3IOREmNA08mquXwUAQJcQciJMVmr74GvO5AAA0DWEnAhzdNVjQg4AAF1ByIkwoe6qA/UtamkL2lwNAADRi5ATYdL7ueV2tR+WmjrO5gAAcKYIORHG4XAoMzQuh8HHAACcMUJOBLIu1Mm4HAAAzhghJwJlHnM1cgAAcGYIOREoixlWAAB0GSEnAmVZqx4321wJAADRi5ATgTJZ9RgAgC4j5ESgLK5fBQBAlxFyItCxs6v68EXiAQDoEkJOBAp1V7W0BfXN4VabqwEAIDoRciKQO86pAf3ckphGDgDAmSLkRCgu1AkAQNcQciIUg48BAOgaQk6EYtVjAAC6hpAToVj1GACAriHkRKis0JXICTkAAJwRQk6EykpNlER3FQAAZ4qQE6GOXRAQAAB0HiEnQoVCTu3hVjW1BmyuBgCA6EPIiVDexDglxLcfHgYfAwDQeYScCOVwOI52WTEuBwCATiPkRLBMxuUAAHDGCDkRLLTqMd1VAAB0HiEnglmXdvA121wJAADRh5ATwY5OI2+0uRIAAKIPISeCMfAYAIAzR8iJYJnWmBy6qwAA6CxCTgQ79iKdwaCxuRoAAKILISeCDUrxyOGQ2oJGBxta7C4HAICoQsiJYPEupwYmt1+NnGnkAAB0DiEnwjH4GACAM0PIiXDWWjmcyQEAoFMIORGOMzkAAJwZQk6E40wOAABnhpAT4TK9XL8KAIAz0emQs2nTJl1zzTXKzs6Ww+HQa6+9Frb9Bz/4gRwOR9jtqquuCmtz6NAh3XrrrfJ6vUpLS9O8efNUX18f1ubjjz/WJZdcooSEBOXk5Gjp0qXH1bJy5UqNHTtWCQkJmjBhgt58883OfpyIR3cVAABnptMhp6GhQZMmTdKyZctO2uaqq67Svn37rNvvfve7sO233nqrduzYofXr12v16tXatGmTfvjDH1rb/X6/pk+fruHDh6usrEyPP/64Hn30Uf3617+22mzevFmzZ8/WvHnz9Je//EWzZs3SrFmztH379s5+pIiWldo+hZzuKgAAOsdhjDnjpXQdDodWrVqlWbNmWc/94Ac/UG1t7XFneEI+/fRTjRs3Tu+//76mTJkiSVq7dq3+/u//Xl999ZWys7P13HPP6Sc/+YmqqqrkdrslSQ8++KBee+017dy5U5J00003qaGhQatXr7b2feGFF+q8887T888/36H6/X6/UlNT5fP55PV6z+Ab6Hl1Ta2a8OifJEmf/CxfSe44mysCAMBeHf397pExORs3blRGRobGjBmjH/3oRzp48KC1rbS0VGlpaVbAkaRp06bJ6XRq69atVptLL73UCjiSlJ+fr4qKCn3zzTdWm2nTpoW9b35+vkpLS3viI9kmJSFeyZ72YEOXFQAAHdftIeeqq67Sb3/7WxUXF+uXv/ylSkpKNGPGDAUCAUlSVVWVMjIywl4TFxen9PR0VVVVWW0yMzPD2oQen65NaPuJNDc3y+/3h92iQab3SJcVIQcAgA7r9r6Pm2++2bo/YcIETZw4Ud/61re0ceNGXXnlld39dp1SVFSkn/70p7bWcCayUhP0xf4GxuUAANAJPT6F/KyzztLAgQP1+eefS5KysrJUU1MT1qatrU2HDh1SVlaW1aa6ujqsTejx6dqEtp/I4sWL5fP5rNuePXu69uF6SWgaOSEHAICO6/GQ89VXX+ngwYMaPHiwJCkvL0+1tbUqKyuz2rz99tsKBoPKzc212mzatEmtra1Wm/Xr12vMmDHq37+/1aa4uDjsvdavX6+8vLyT1uLxeOT1esNu0SA0jbya7ioAADqs0yGnvr5e5eXlKi8vlyTt2rVL5eXlqqysVH19ve6//35t2bJFu3fvVnFxsa699lqNGjVK+fn5kqRzzjlHV111le666y699957evfddzV//nzdfPPNys7OliTdcsstcrvdmjdvnnbs2KEVK1boqaeeUmFhoVXHvffeq7Vr1+qJJ57Qzp079eijj+qDDz7Q/Pnzu+FriSysegwAwBkwnbRhwwYj6bjbnDlzzOHDh8306dPNoEGDTHx8vBk+fLi56667TFVVVdg+Dh48aGbPnm2Sk5ON1+s1c+fONXV1dWFtPvroI3PxxRcbj8djhgwZYh577LHjann11VfN6NGjjdvtNueee65Zs2ZNpz6Lz+czkozP5+vs19Cr1m7fZ4YvWm3+n2fesbsUAABs19Hf7y6tkxPtomGdHEn6aE+trl32rrK8Cdry/9o7eBsAALvZuk4OutfgI91V++ub1RYI2lwNAADRgZATBQYke+RyOhQIGh2ob7G7HAAAogIhJwq4nA5lpHANKwAAOoOQEyUyuRo5AACdQsiJEtZaOZzJAQCgQwg5UYK1cgAA6BxCTpSguwoAgM4h5ESJYelJkqQvDzbYXAkAANGBkBMlRgxsDzm7Dx62uRIAAKIDISdKjBjQT5J0qKFFvsOtp2kNAAAIOVGinyfOWitnF11WAACcFiEniowc2H42Z/cBQg4AAKdDyIkioZCzi5ADAMBpEXKiyAhCDgAAHUbIiSJWdxVjcgAAOC1CThQ5trvKGGNzNQAARDZCThQZlp4kh0Oqa2rToYYWu8sBACCiEXKiSEK8S9mpiZIYlwMAwOkQcqIMM6wAAOgYQk6UOXp5B0IOAACnQsiJMqHLO3AmBwCAUyPkRJmj3VVcqBMAgFMh5ESZUMj58iDTyAEAOBVCTpTJSU+Sy+nQ4ZaAauqa7S4HAICIRciJMvEup4b2b59G/tf9jMsBAOBkCDlRKDT4mBlWAACcHCEnClnXsGKGFQAAJ0XIiUIsCAgAwOkRcqLQCEIOAACnRciJQiOPjMn58tBhBYNMIwcA4EQIOVFoSP9ExbscamkLaq+v0e5yAACISIScKORyOjQs/cg1rFj5GACAEyLkRKmjg4/rba4EAIDIRMiJUkcv1MmZHAAAToSQE6VGDmJBQAAAToWQE6VGDmAaOQAAp0LIiVKhtXL2HDqstkDQ5moAAIg8hJwoleVNUEK8U21Bo6++YRo5AAB/i5ATpZxOx9HBx4zLAQDgOIScKGaFnP2EHAAA/hYhJ4qFxuUwwwoAgON1OuRs2rRJ11xzjbKzs+VwOPTaa6+FbTfGaMmSJRo8eLASExM1bdo0ffbZZ2FtDh06pFtvvVVer1dpaWmaN2+e6uvDF7X7+OOPdckllyghIUE5OTlaunTpcbWsXLlSY8eOVUJCgiZMmKA333yzsx8nqp3FhToBADipToechoYGTZo0ScuWLTvh9qVLl+pXv/qVnn/+eW3dulX9+vVTfn6+mpqarDa33nqrduzYofXr12v16tXatGmTfvjDH1rb/X6/pk+fruHDh6usrEyPP/64Hn30Uf3617+22mzevFmzZ8/WvHnz9Je//EWzZs3SrFmztH379s5+pKjFmRwAAE7BdIEks2rVKutxMBg0WVlZ5vHHH7eeq62tNR6Px/zud78zxhjzySefGEnm/ffft9q89dZbxuFwmK+//toYY8yzzz5r+vfvb5qbm602ixYtMmPGjLEe/8M//IOZOXNmWD25ubnmH//xHztcv8/nM5KMz+fr8GsiSbW/0QxftNqMfHC1aWpts7scAAB6RUd/v7t1TM6uXbtUVVWladOmWc+lpqYqNzdXpaWlkqTS0lKlpaVpypQpVptp06bJ6XRq69atVptLL71UbrfbapOfn6+Kigp98803Vptj3yfUJvQ+J9Lc3Cy/3x92i2aDkj3q53YpaNrXywEAAEd1a8ipqqqSJGVmZoY9n5mZaW2rqqpSRkZG2Pa4uDilp6eHtTnRPo59j5O1CW0/kaKiIqWmplq3nJyczn7EiOJwOKzLO3ANKwAAwvWp2VWLFy+Wz+ezbnv27LG7pC4LTSPfzeBjAADCdGvIycrKkiRVV1eHPV9dXW1ty8rKUk1NTdj2trY2HTp0KKzNifZx7HucrE1o+4l4PB55vd6wW7QbeWTw8V8JOQAAhOnWkDNy5EhlZWWpuLjYes7v92vr1q3Ky8uTJOXl5am2tlZlZWVWm7ffflvBYFC5ublWm02bNqm1tdVqs379eo0ZM0b9+/e32hz7PqE2offpKziTAwDAiXU65NTX16u8vFzl5eWS2gcbl5eXq7KyUg6HQwsWLNAvfvELvf7669q2bZtuv/12ZWdna9asWZKkc845R1dddZXuuusuvffee3r33Xc1f/583XzzzcrOzpYk3XLLLXK73Zo3b5527NihFStW6KmnnlJhYaFVx7333qu1a9fqiSee0M6dO/Xoo4/qgw8+0Pz587v+rUSR0JgcppEDAPA3Ojtta8OGDUbScbc5c+YYY9qnkT/88MMmMzPTeDwec+WVV5qKioqwfRw8eNDMnj3bJCcnG6/Xa+bOnWvq6urC2nz00Ufm4osvNh6PxwwZMsQ89thjx9Xy6quvmtGjRxu3223OPfdcs2bNmk59lmifQm6MMYfqm83wRavN8EWrzeFmppEDAGJfR3+/HcYYY2PGspXf71dqaqp8Pl9Uj8+Z9NM/ydfYqrfuvUTnDI7ezwEAQEd09Pe7T82uilXWyseMywEAwELIiQHWNawYlwMAgIWQEwNCM6x27SfkAAAQQsiJASMGJklihhUAAMci5MSA0IKAXNoBAICjCDkxIDTw+EB9s+qaWk/TGgCAvoGQEwO8CfEamNx+xfbdnM0BAEASISdmWIOPGZcDAIAkQk7MGMlaOQAAhCHkxIgR1uBjQg4AABIhJ2aMJOQAABCGkBMjQmNyWCsHAIB2hJwYEVoQsPZwq75paLG5GgAA7EfIiRFJ7jhleRMkMcMKAACJkBNTrMs7MC4HAABCTixhGjkAAEcRcmKINcPqIKseAwBAyIkh1qrHB+ptrgQAAPsRcmLI0e6qwzLG2FwNAAD2IuTEkJz0JDkcUn1zmw7UM40cANC3EXJiSEK8S0PSEiWxKCAAAIScGGMNPt5PyAEA9G2EnBhjDT7mTA4AoI8j5MSYEayVAwCAJEJOzDmLq5EDACCJkBNzrDM5BxsUDDKNHADQdxFyYszQ/olyOR1qag2quq7J7nIAALANISfGxLucyunfPo2cLisAQF9GyIlB3xqULEkq2/2NzZUAAGAfQk4MmjFhsCTpd+9Vqi0QtLkaAADsQciJQVdPHKz0fm7t9TVp/SfVdpcDAIAtCDkxKCHepdlTcyRJyzfvtrcYAABsQsiJUbfmDpfL6dDWXYe0s8pvdzkAAPQ6Qk6Myk5L1PRxmZKklzZ/aXM1AAD0PkJODJvzdyMkSa/95Wv5DrfaWwwAAL2MkBPDckema2xWihpbA3r1gz12lwMAQK8i5MQwh8Nhnc35jy1fKsBlHgAAfQghJ8bNOm+IUhPjVXnosDZW1NhdDgAAvYaQE+MS3S7ddAHTyQEAfQ8hpw/4fu5wORzSnz87oC/219tdDgAAvYKQ0wcMG5CkK8dmSJL+o5Tp5ACAvqHbQ86jjz4qh8MRdhs7dqy1vampSQUFBRowYICSk5N1ww03qLo6/NIDlZWVmjlzppKSkpSRkaH7779fbW1tYW02btyo888/Xx6PR6NGjdLy5cu7+6PElNAA5N+XfaX65rZTNwYAIAb0yJmcc889V/v27bNu77zzjrVt4cKFeuONN7Ry5UqVlJRo7969uv76663tgUBAM2fOVEtLizZv3qyXXnpJy5cv15IlS6w2u3bt0syZM3XFFVeovLxcCxYs0J133ql169b1xMeJCRePGqhvDeqn+uY2/XfZV3aXAwBAj3MYY7p1XvGjjz6q1157TeXl5cdt8/l8GjRokF5++WV973vfkyTt3LlT55xzjkpLS3XhhRfqrbfe0tVXX629e/cqM7N9xd7nn39eixYt0v79++V2u7Vo0SKtWbNG27dvt/Z98803q7a2VmvXru1wrX6/X6mpqfL5fPJ6vV374FHgt6W7teSPO3TWoH76n4WXyel02F0SAACd1tHf7x45k/PZZ58pOztbZ511lm699VZVVlZKksrKytTa2qpp06ZZbceOHathw4aptLRUklRaWqoJEyZYAUeS8vPz5ff7tWPHDqvNsfsItQnt42Sam5vl9/vDbn3J9ecPVbInTn/d36B3Pj9gdzkAAPSobg85ubm5Wr58udauXavnnntOu3bt0iWXXKK6ujpVVVXJ7XYrLS0t7DWZmZmqqqqSJFVVVYUFnND20LZTtfH7/WpsbDxpbUVFRUpNTbVuOTk5Xf24USXZE6fvTR4qqf2sDgAAsazbQ86MGTN04403auLEicrPz9ebb76p2tpavfrqq939Vp22ePFi+Xw+67ZnT9+71MHtecMlScU7a1R58LDN1QAA0HN6fAp5WlqaRo8erc8//1xZWVlqaWlRbW1tWJvq6mplZWVJkrKyso6bbRV6fLo2Xq9XiYmJJ63F4/HI6/WG3fqaswYl69LRg2SM9B9bdttdDgAAPabHQ059fb2++OILDR48WJMnT1Z8fLyKi4ut7RUVFaqsrFReXp4kKS8vT9u2bVNNzdFLEKxfv15er1fjxo2z2hy7j1Cb0D5waj/4u/azOSve36PDLUwnBwDEpm4POffdd59KSkq0e/dubd68Wdddd51cLpdmz56t1NRUzZs3T4WFhdqwYYPKyso0d+5c5eXl6cILL5QkTZ8+XePGjdNtt92mjz76SOvWrdNDDz2kgoICeTweSdLdd9+tv/71r3rggQe0c+dOPfvss3r11Ve1cOHC7v44Meny0Rkalp4kf1Ob/li+1+5yAADoEd0ecr766ivNnj1bY8aM0T/8wz9owIAB2rJliwYNGiRJ+rd/+zddffXVuuGGG3TppZcqKytLf/jDH6zXu1wurV69Wi6XS3l5efr+97+v22+/XT/72c+sNiNHjtSaNWu0fv16TZo0SU888YReeOEF5efnd/fHiUlOp8Mam/PS5t3q5lUEAACICN2+Tk406Wvr5BzL19iqC/+/YjW2BvTKDy/UhWcNsLskAAA6xNZ1chD5UhPjdd35QyS1n80BACDWEHL6sDl5IyRJf/qkWhVVdfYWAwBANyPk9GFjslI0fVymAkGjB37/kdoCQbtLAgCg2xBy+rifzxqvlIQ4ffSVT//n3V12lwMAQLch5PRxmd4EPTyzff2hJ/70v/rr/nqbKwIAoHsQcqAbpwzVJWcPVHNbUA/+9zYFg312wh0AIIYQciCHw6Gi6yeon9ul93Yf0n9u/dLukgAA6DJCDiRJQ/snadGMsZKkx97aqT2HuHgnACC6EXJg+X7ucE0dka7DLQEt/sM2VkIGAEQ1Qg4sTqdDv/zeRHninHrn8wN69YM9dpcEAMAZI+QgzMiB/XTf9DGSpF+s+VRVviabKwIA4MwQcnCcOy4eqUk5aapratNPVtFtBQCIToQcHMfldOjx701UvMuh4p01ev2jvXaXBABApxFycEKjM1P0z985W5L06Os7dKC+2eaKAADoHEIOTuruy7+lcwZ79c3hVj3y+g67ywEAoFMIOTipeJdTj39volxOh9Z8vE9rt++zuyQAADqMkINTGj8kVXdfdpYk6aHXdqj2cIvNFQEA0DGEHJzWPd85W6MyknWgvlkLV5TLd7jV7pIAADgtQg5OKyHepaXfm6g4p0MbKvZr+pMl2lBRY3dZAACcEiEHHXL+sP5a8Y95OmtgP1X7mzX3xfe16Pcfq66JszoAgMhEyEGHTR7eX2v++RLdcdFIORzSig/26Kon/6x3Pz9gd2kAAByHkINOSXS7tOSacXrlrgs1LD1JX9c26tYXtuqh17apobnN7vIAALAQcnBGcs8aoLfuvUS3XThckvSfWyo146k/a+tfD9pcGQAA7Qg5OGP9PHH6+azx+s95uRqSlqjKQ4d182+26GdvfKLGloDd5QEA+jhCDrrs4rMHau2CS3TzBTkyRvo/7+7SzF/9WcWfVnNxTwCAbRymD/8K+f1+paamyufzyev12l1OTNhQUaMH//tjVfvbr3U1NitFP7r8W5o5YbDiXGRqAEDXdfT3m5BDyOl2vsOterbkc/3XlkrVHxmMPCw9Sf942Vm64fyhSoh32VwhACCaEXI6gJDTs3yHW/Xb0t16cfNuHWpovxxERopHd14yUrfkDleyJ87mCgEA0YiQ0wGEnN5xuKVNK97fo19v+qv2+ZokSamJ8ZqTN1w/uGik0vu5ba4QABBNCDkdQMjpXS1tQb1W/rWeL/lCf93fIElKjHfppgtydGvuMJ2dmWJzhQCAaEDI6QBCjj0CQaN1O6r07MbPtf1rv/X8eTlpunHKUF0zKVvehHgbKwQARDJCTgcQcuxljNGfPzug35Z+qQ0VNQoE2/9T9MQ5ddX4LN04OUd/960BcjodNlcKAIgkhJwOIOREjv11zXrtL19rZdke/W91vfX8kLRE3XD+EH1vco6GDUiysUIAQKQg5HQAISfyGGP08Vc+rSzbo9fL98rfdPR6WLkj03X1xMEaPyRVY7O8SnQzFR0A+iJCTgcQciJbU2tAf/qkWis/2KN3Pj+gY/9LdTqkkQP76dzsVI3L9mrcYK/OzfZqQLLHvoIBAL2CkNMBhJzosbe2Uav+8rW27jqkT/b6dKC+5YTtMr0ejRvs1bhsr87OSNGwAUkaMaCf+ifFy+FgbA8AxAJCTgcQcqJXTV2Tduz165O9fn2yr/3vrgMNJ22fkhCnEQP6HQk9SRo+oJ+GpydpxMB+ykjxEIAAIIoQcjqAkBNb6pvbVFHlt8LPrgMN+vLgYVX5m075uoR4p7K8Ccr0JigrNSHsfuhvRopH8Vx7CwAiAiGnAwg5fUNTa0CVhw5r95HQ8+Wh9r+7Dzbo628aFezAvwCHQxrQz61Mb3vgGZTiUUZKgjK8Hg1K9ijD2/54UIqHa3MBQA/r6O83Fw9CzEuId2l0ZopGn2BF5Za2oPbWNqrK36Rqf5OqfE3W/Wp/s6p8Taqpa1JrwOhAfYsO1Ldox2neLyUhThkpHvVPcivJE6d+bpeS3HHq5zny1+06+vyRv4lH2iS5XUqMdynpyOOEeCddaQBwhqI+5CxbtkyPP/64qqqqNGnSJD399NOaOnWq3WUhSrjjnBoxsJ9GDOx30jbBoNGhwy1W4Nlf16waf7Nq6prb79c1qaau/XFLW1B1TW2qa2qTdPIxQp0RCj2JbpeSPXEalOJRpjdBmd7Q3yPdat4EDUx2K45uNQCQFOUhZ8WKFSosLNTzzz+v3NxcPfnkk8rPz1dFRYUyMjLsLg8xwul0aGCyRwOTPZJST9rOGCN/U5sVfPyNrWpoDuhwS5saWgI63Hzkb0vb0eebA2poadPhloAaj2xrbA2oqTVo7bexNaDG1oCVmXZW1Z20BodDGpjsUZY3QWlHZpQ5jjwv6cj99udkPe+QyynFuZyKdzra/7occjkdinO23w9ti3c5leSJU0pCnFI8cUpJiG+/nxCn5IQ4eRPi5Ynj7BOAyBDVY3Jyc3N1wQUX6JlnnpEkBYNB5eTk6J577tGDDz542tczJgeRKhg0amwNHA0/rW1qbAmorqlNNXXNqvY3qcYf6lo78riu2bo0hp3iXQ6lJMQrye1SnNMhp8Mhh6M9XDkdOvL46H2noz1Ixjud7cHK1R6m4o7cj3M6259zOuVyOax9upztt/b7ksvhkNPpOPrX2d423nX09XHHBLZQmIt3ORXvcsoT51RCvMv6mxDvlCeu/TGXFgEiS8yPyWlpaVFZWZkWL15sPed0OjVt2jSVlpbaWBnQdU6nQ/08cern6fg/0UDQ6GBDe1dala9J/qZWGSOFYo8xpv3+kSeMjLU9EDRqCwTVFjRqDbTfbw17LqjAkb/1zQHVNbWqrqlN9U1tR++3tMkYqTVgdKihRYe6p7cuIrjjjoYg95HuQKdTcqg9pDlCQU6yQl0oiMW5nHIfE9bcodB1JFzFOR2Kj3O2h7Mjgc915LWOYwKc4wTB7viwdzQ8uo4JmMcKnWU79mzeh1/Wat2OKknH/HciWQtwhp4xRvr2sDTdPHWY4k4T/Mora9XQEujqVx+ROFHZcbfnDdfg1ETb3j9qQ86BAwcUCASUmZkZ9nxmZqZ27tx5wtc0NzerubnZeuz3+0/YDohGLqejfcZXSoLGDzl5t1pPCQaN6ltCwadN9c1tMsYoaKSgMQqa9lB1or9tQWOFqLbAkfvB9vttfxO2gkGjgDEKBNtfGzjy2tD9o89JbUf20XrM69v3GVTLkTAX2t7cFlRTa0BNrQE1t7W3D2lpC1rjrfq6dTuqtW5Htd1lIEr8e8kX2vnzGXLH2TNWMGpDzpkoKirST3/6U7vLAGKS0+mQNyFe3oR4u0vpFm3HBp+2oJqPjJVqDQRlJCvASe1/jw1uoW1tweNDVWsgePRsWaA9zLW2GSsIBo6EOGNkBThjjgl2R7YfDXvhwe7YsNd+pu7oWZiwv0eeb2gOaNvXvk59N26XU9/KSD5lm0/38T+RkK49b4htAUeK4pAzcOBAuVwuVVeH/x9FdXW1srKyTviaxYsXq7Cw0Hrs9/uVk5PTo3UCiE5xLqfiXM5OdRkCiCxRO9fU7XZr8uTJKi4utp4LBoMqLi5WXl7eCV/j8Xjk9XrDbgAAIDZF9f+iFBYWas6cOZoyZYqmTp2qJ598Ug0NDZo7d67dpQEAAJtFdci56aabtH//fi1ZskRVVVU677zztHbt2uMGIwMAgL4nqtfJ6SrWyQEAIPp09Pc7asfkAAAAnAohBwAAxCRCDgAAiEmEHAAAEJMIOQAAICYRcgAAQEwi5AAAgJhEyAEAADGJkAMAAGISIQcAAMSkqL52VVeFrmjh9/ttrgQAAHRU6Hf7dFem6tMhp66uTpKUk5NjcyUAAKCz6urqlJqaetLtffoCncFgUHv37lVKSoocDscp2/r9fuXk5GjPnj1czDMKcLyiB8cqenCsokssHy9jjOrq6pSdnS2n8+Qjb/r0mRyn06mhQ4d26jVerzfm/mOJZRyv6MGxih4cq+gSq8frVGdwQhh4DAAAYhIhBwAAxCRCTgd5PB498sgj8ng8dpeCDuB4RQ+OVfTgWEUXjlcfH3gMAABiF2dyAABATCLkAACAmETIAQAAMYmQ00HLli3TiBEjlJCQoNzcXL333nt2lwRJmzZt0jXXXKPs7Gw5HA699tprYduNMVqyZIkGDx6sxMRETZs2TZ999pk9xfZhRUVFuuCCC5SSkqKMjAzNmjVLFRUVYW2amppUUFCgAQMGKDk5WTfccIOqq6ttqrhve+655zRx4kRrfZW8vDy99dZb1naOVeR67LHH5HA4tGDBAuu5vny8CDkdsGLFChUWFuqRRx7Rhx9+qEmTJik/P181NTV2l9bnNTQ0aNKkSVq2bNkJty9dulS/+tWv9Pzzz2vr1q3q16+f8vPz1dTU1MuV9m0lJSUqKCjQli1btH79erW2tmr69OlqaGiw2ixcuFBvvPGGVq5cqZKSEu3du1fXX3+9jVX3XUOHDtVjjz2msrIyffDBB/rOd76ja6+9Vjt27JDEsYpU77//vv793/9dEydODHu+Tx8vg9OaOnWqKSgosB4HAgGTnZ1tioqKbKwKf0uSWbVqlfU4GAyarKws8/jjj1vP1dbWGo/HY373u9/ZUCFCampqjCRTUlJijGk/LvHx8WblypVWm08//dRIMqWlpXaViWP079/fvPDCCxyrCFVXV2fOPvtss379enPZZZeZe++91xjDvy3O5JxGS0uLysrKNG3aNOs5p9OpadOmqbS01MbKcDq7du1SVVVV2LFLTU1Vbm4ux85mPp9PkpSeni5JKisrU2tra9ixGjt2rIYNG8axslkgENArr7yihoYG5eXlcawiVEFBgWbOnBl2XCT+bfXpa1d1xIEDBxQIBJSZmRn2fGZmpnbu3GlTVeiIqqoqSTrhsQttQ+8LBoNasGCBLrroIo0fP15S+7Fyu91KS0sLa8uxss+2bduUl5enpqYmJScna9WqVRo3bpzKy8s5VhHmlVde0Ycffqj333//uG19/d8WIQdAryooKND27dv1zjvv2F0KTmHMmDEqLy+Xz+fT73//e82ZM0clJSV2l4W/sWfPHt17771av369EhIS7C4n4tBddRoDBw6Uy+U6biR6dXW1srKybKoKHRE6Phy7yDF//nytXr1aGzZs0NChQ63ns7Ky1NLSotra2rD2HCv7uN1ujRo1SpMnT1ZRUZEmTZqkp556imMVYcrKylRTU6Pzzz9fcXFxiouLU0lJiX71q18pLi5OmZmZffp4EXJOw+12a/LkySouLraeCwaDKi4uVl5eno2V4XRGjhyprKyssGPn9/u1detWjl0vM8Zo/vz5WrVqld5++22NHDkybPvkyZMVHx8fdqwqKipUWVnJsYoQwWBQzc3NHKsIc+WVV2rbtm0qLy+3blOmTNGtt95q3e/Lx4vuqg4oLCzUnDlzNGXKFE2dOlVPPvmkGhoaNHfuXLtL6/Pq6+v1+eefW4937dql8vJypaena9iwYVqwYIF+8Ytf6Oyzz9bIkSP18MMPKzs7W7NmzbKv6D6ooKBAL7/8sv74xz8qJSXFGguQmpqqxMREpaamat68eSosLFR6erq8Xq/uuece5eXl6cILL7S5+r5n8eLFmjFjhoYNG6a6ujq9/PLL2rhxo9atW8exijApKSnW2LaQfv36acCAAdbzffp42T29K1o8/fTTZtiwYcbtdpupU6eaLVu22F0SjDEbNmwwko67zZkzxxjTPo384YcfNpmZmcbj8Zgrr7zSVFRU2Ft0H3SiYyTJvPjii1abxsZG80//9E+mf//+JikpyVx33XVm37599hXdh91xxx1m+PDhxu12m0GDBpkrr7zS/OlPf7K2c6wi27FTyI3p28eLq5ADAICYxJgcAAAQkwg5AAAgJhFyAABATCLkAACAmETIAQAAMYmQAwAAYhIhBwAAxCRCDgAAiEmEHAA4jR/84AdcCgSIQoQcAAAQkwg5APq0QCCgYDBodxkAegAhB0CXrV27VhdffLHS0tI0YMAAXX311friiy8kSbt375bD4dAf/vAHXXHFFUpKStKkSZNUWlpqvf7LL7/UNddco/79+6tfv34699xz9eabb0qSpkyZon/913+12s6aNUvx8fGqr6+XJH311VdyOBzW1eibm5t13333aciQIerXr59yc3O1ceNG6/XLly9XWlqaXn/9dY0bN04ej0eVlZWd+rzBYFBFRUUaOXKkEhMTNWnSJP3+97+3tm/cuFEOh0PFxcWaMmWKkpKS9Hd/93eqqKjo3BcLoEsIOQC6rKGhQYWFhfrggw9UXFwsp9Op6667LuwMyU9+8hPdd999Ki8v1+jRozV79my1tbVJkgoKCtTc3KxNmzZp27Zt+uUvf6nk5GRJ0mWXXWaFFGOM/vznPystLU3vvPOOJKmkpERDhgzRqFGjJEnz589XaWmpXnnlFX388ce68cYbddVVV+mzzz6zajl8+LB++ctf6oUXXtCOHTuUkZHRqc9bVFSk3/72t3r++ee1Y8cOLVy4UN///vdVUlIS1u4nP/mJnnjiCX3wwQeKi4vTHXfc0bkvFkDX2HwVdAAxaP/+/UaS2bZtm9m1a5eRZF544QVr+44dO4wk8+mnnxpjjJkwYYJ59NFHT7iv119/3aSmppq2tjZTXl5usrKyzL333msWLVpkjDHmzjvvNLfccosxxpgvv/zSuFwu8/XXX4ft48orrzSLFy82xhjz4osvGkmmvLy8w59nzpw55tprrzXGGNPU1GSSkpLM5s2bw9rMmzfPzJ492xhjzIYNG4wk8z//8z/W9jVr1hhJprGxscPvC6BrOJMDoMs+++wzzZ49W2eddZa8Xq9GjBghSWHdQBMnTrTuDx48WJJUU1MjSfrnf/5n/eIXv9BFF12kRx55RB9//LHV9pJLLlFdXZ3+8pe/qKSkRJdddpkuv/xy6+xOSUmJLr/8cknStm3bFAgENHr0aCUnJ1u3kpISq/tMktxud1g9nfH555/r8OHD+u53vxv2Hr/97W/D3uN0nxlAz4uzuwAA0e+aa67R8OHD9Zvf/EbZ2dkKBoMaP368WlparDbx8fHWfYfDIUlWd9add96p/Px8rVmzRn/6059UVFSkJ554Qvfcc4/S0tI0adIkbdy4UaWlpfrud7+rSy+9VDfddJP+93//V5999pkuu+wySVJ9fb1cLpfKysrkcrnCagx1f0lSYmKiVUNnhcYCrVmzRkOGDAnb5vF4wh6f6jMD6HmEHABdcvDgQVVUVOg3v/mNLrnkEkmyxst0Rk5Oju6++27dfffdWrx4sX7zm9/onnvukdQ+LmfDhg1677339C//8i9KT0/XOeeco3/5l3/R4MGDNXr0aEnSt7/9bQUCAdXU1Fi1dLdjByuHwhWAyETIAdAl/fv314ABA/TrX/9agwcPVmVlpR588MFO7WPBggWaMWOGRo8erW+++UYbNmzQOeecY22//PLL9fTTT2vQoEEaO3as9dwzzzyjG2+80Wo3evRo3Xrrrbr99tv1xBNP6Nvf/rb279+v4uJiTZw4UTNnzuzy501JSdF9992nhQsXKhgM6uKLL5bP59O7774rr9erOXPmdPk9AHQPxuQA6BKn06lXXnlFZWVlGj9+vBYuXKjHH3+8U/sIBAIqKCjQOeeco6uuukqjR4/Ws88+a22/5JJLFAwGw86cXH755QoEAtZ4nJAXX3xRt99+u3784x9rzJgxmjVrlt5//30NGzasS5/zWD//+c/18MMPq6ioyKp5zZo1GjlyZLe9B4CucxhjjN1FAAAAdDfO5AAAgJhEyAHQ5x07Ffxvb3/+85/tLg/AGaK7CkCfF7okxIkMGTJEiYmJvVgNgO5CyAEAADGJ7ioAABCTCDkAACAmEXIAAEBMIuQAAICYRMgBAAAxiZADAABiEiEHAADEJEIOAACISf8XgZgxD7sRVnAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df[\"answer_len\"] = df[\"answer\"].apply(lambda x: len(str(x).split()))\n",
    "df.answer_len.value_counts().plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answer_sent</th>\n",
       "      <th>answer_chunk</th>\n",
       "      <th>answer</th>\n",
       "      <th>answer_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5733be284776f41900661182</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>It is a replica of the grotto at Lourdes, Fran...</td>\n",
       "      <td>Atop the Main Building's gold dome is a golden...</td>\n",
       "      <td>Saint Bernadette Soubirous</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5733be284776f41900661180</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "      <td>Atop the Main Building's gold dome is a golden...</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>the Main Building</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5733bf84d058e614000b61be</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>As at most other universities, Notre Dame's st...</td>\n",
       "      <td>When did the Scholastic Magazine of Notre dame...</td>\n",
       "      <td>Begun as a one-page journal in September 1876,...</td>\n",
       "      <td>As at most other universities, Notre Dame's st...</td>\n",
       "      <td>September 1876</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5733bf84d058e614000b61bf</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>As at most other universities, Notre Dame's st...</td>\n",
       "      <td>How often is Notre Dame's the Juggler published?</td>\n",
       "      <td>Begun as a one-page journal in September 1876,...</td>\n",
       "      <td>As at most other universities, Notre Dame's st...</td>\n",
       "      <td>twice</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5733bf84d058e614000b61c0</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>As at most other universities, Notre Dame's st...</td>\n",
       "      <td>What is the daily student paper at Notre Dame ...</td>\n",
       "      <td>The newspapers have varying publication intere...</td>\n",
       "      <td>The newspapers have varying publication intere...</td>\n",
       "      <td>The Observer</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92059</th>\n",
       "      <td>5737a9afc3c5551400e51f63</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What is the exchange of heat associated with?</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>nonconservative forces</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92060</th>\n",
       "      <td>5737a9afc3c5551400e51f65</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What makes energy changes in a closed system?</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>nonconservative forces</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92061</th>\n",
       "      <td>5737aafd1c456719005744fc</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What is the kilogram-force sometimes reffered ...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>kilopond</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92062</th>\n",
       "      <td>5737aafd1c456719005744fe</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What seldom used term of a unit of force equal...</td>\n",
       "      <td>Other arcane units of force include the sthène...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>kip</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92063</th>\n",
       "      <td>5737aafd1c456719005744ff</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What is the seldom used force unit equal to on...</td>\n",
       "      <td>Other arcane units of force include the sthène...</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>sthène</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76397 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             id                     title  \\\n",
       "0      5733be284776f41900661182  University_of_Notre_Dame   \n",
       "2      5733be284776f41900661180  University_of_Notre_Dame   \n",
       "5      5733bf84d058e614000b61be  University_of_Notre_Dame   \n",
       "6      5733bf84d058e614000b61bf  University_of_Notre_Dame   \n",
       "7      5733bf84d058e614000b61c0  University_of_Notre_Dame   \n",
       "...                         ...                       ...   \n",
       "92059  5737a9afc3c5551400e51f63                     Force   \n",
       "92060  5737a9afc3c5551400e51f65                     Force   \n",
       "92061  5737aafd1c456719005744fc                     Force   \n",
       "92062  5737aafd1c456719005744fe                     Force   \n",
       "92063  5737aafd1c456719005744ff                     Force   \n",
       "\n",
       "                                                 context  \\\n",
       "0      Architecturally, the school has a Catholic cha...   \n",
       "2      Architecturally, the school has a Catholic cha...   \n",
       "5      As at most other universities, Notre Dame's st...   \n",
       "6      As at most other universities, Notre Dame's st...   \n",
       "7      As at most other universities, Notre Dame's st...   \n",
       "...                                                  ...   \n",
       "92059  The connection between macroscopic nonconserva...   \n",
       "92060  The connection between macroscopic nonconserva...   \n",
       "92061  The pound-force has a metric counterpart, less...   \n",
       "92062  The pound-force has a metric counterpart, less...   \n",
       "92063  The pound-force has a metric counterpart, less...   \n",
       "\n",
       "                                                question  \\\n",
       "0      To whom did the Virgin Mary allegedly appear i...   \n",
       "2      The Basilica of the Sacred heart at Notre Dame...   \n",
       "5      When did the Scholastic Magazine of Notre dame...   \n",
       "6       How often is Notre Dame's the Juggler published?   \n",
       "7      What is the daily student paper at Notre Dame ...   \n",
       "...                                                  ...   \n",
       "92059      What is the exchange of heat associated with?   \n",
       "92060      What makes energy changes in a closed system?   \n",
       "92061  What is the kilogram-force sometimes reffered ...   \n",
       "92062  What seldom used term of a unit of force equal...   \n",
       "92063  What is the seldom used force unit equal to on...   \n",
       "\n",
       "                                             answer_sent  \\\n",
       "0      It is a replica of the grotto at Lourdes, Fran...   \n",
       "2      Atop the Main Building's gold dome is a golden...   \n",
       "5      Begun as a one-page journal in September 1876,...   \n",
       "6      Begun as a one-page journal in September 1876,...   \n",
       "7      The newspapers have varying publication intere...   \n",
       "...                                                  ...   \n",
       "92059  The connection between macroscopic nonconserva...   \n",
       "92060  The connection between macroscopic nonconserva...   \n",
       "92061  The pound-force has a metric counterpart, less...   \n",
       "92062  Other arcane units of force include the sthène...   \n",
       "92063  Other arcane units of force include the sthène...   \n",
       "\n",
       "                                            answer_chunk  \\\n",
       "0      Atop the Main Building's gold dome is a golden...   \n",
       "2      Architecturally, the school has a Catholic cha...   \n",
       "5      As at most other universities, Notre Dame's st...   \n",
       "6      As at most other universities, Notre Dame's st...   \n",
       "7      The newspapers have varying publication intere...   \n",
       "...                                                  ...   \n",
       "92059  The connection between macroscopic nonconserva...   \n",
       "92060  The connection between macroscopic nonconserva...   \n",
       "92061  The pound-force has a metric counterpart, less...   \n",
       "92062  The pound-force has a metric counterpart, less...   \n",
       "92063  The pound-force has a metric counterpart, less...   \n",
       "\n",
       "                           answer  answer_len  \n",
       "0      Saint Bernadette Soubirous           3  \n",
       "2               the Main Building           3  \n",
       "5                  September 1876           2  \n",
       "6                           twice           1  \n",
       "7                    The Observer           2  \n",
       "...                           ...         ...  \n",
       "92059      nonconservative forces           2  \n",
       "92060      nonconservative forces           2  \n",
       "92061                    kilopond           1  \n",
       "92062                         kip           1  \n",
       "92063                      sthène           1  \n",
       "\n",
       "[76397 rows x 8 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.answer_len < 5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6ac0d3e290c431baa5dd034d0ac4525",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/77 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[57], line 6\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mlen\u001b[39m(texts), BATCH_SIZE)):\n\u001b[1;32m      5\u001b[0m     batch \u001b[38;5;241m=\u001b[39m texts[i:i\u001b[38;5;241m+\u001b[39mBATCH_SIZE]\n\u001b[0;32m----> 6\u001b[0m     docs\u001b[38;5;241m.\u001b[39mextend(\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mner\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpipe\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/language.py:1618\u001b[0m, in \u001b[0;36mLanguage.pipe\u001b[0;34m(self, texts, as_tuples, batch_size, disable, component_cfg, n_process)\u001b[0m\n\u001b[1;32m   1616\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m pipe \u001b[38;5;129;01min\u001b[39;00m pipes:\n\u001b[1;32m   1617\u001b[0m         docs \u001b[38;5;241m=\u001b[39m pipe(docs)\n\u001b[0;32m-> 1618\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m doc \u001b[38;5;129;01min\u001b[39;00m docs:\n\u001b[1;32m   1619\u001b[0m     \u001b[38;5;28;01myield\u001b[39;00m doc\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1685\u001b[0m, in \u001b[0;36m_pipe\u001b[0;34m(docs, proc, name, default_error_handler, kwargs)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_pipe\u001b[39m(\n\u001b[1;32m   1676\u001b[0m     docs: Iterable[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   1677\u001b[0m     proc: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeCallable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1682\u001b[0m     kwargs: Mapping[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m   1683\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m   1684\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(proc, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpipe\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m-> 1685\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m proc\u001b[38;5;241m.\u001b[39mpipe(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1686\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1687\u001b[0m         \u001b[38;5;66;03m# We added some args for pipe that __call__ doesn't expect.\u001b[39;00m\n\u001b[1;32m   1688\u001b[0m         kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(kwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/pipeline/transition_parser.pyx:245\u001b[0m, in \u001b[0;36mpipe\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1632\u001b[0m, in \u001b[0;36mminibatch\u001b[0;34m(items, size)\u001b[0m\n\u001b[1;32m   1630\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m   1631\u001b[0m     batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(size_)\n\u001b[0;32m-> 1632\u001b[0m     batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitertools\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mislice\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitems\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1633\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(batch) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1634\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1685\u001b[0m, in \u001b[0;36m_pipe\u001b[0;34m(docs, proc, name, default_error_handler, kwargs)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_pipe\u001b[39m(\n\u001b[1;32m   1676\u001b[0m     docs: Iterable[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   1677\u001b[0m     proc: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeCallable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1682\u001b[0m     kwargs: Mapping[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m   1683\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m   1684\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(proc, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpipe\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m-> 1685\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m proc\u001b[38;5;241m.\u001b[39mpipe(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1686\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1687\u001b[0m         \u001b[38;5;66;03m# We added some args for pipe that __call__ doesn't expect.\u001b[39;00m\n\u001b[1;32m   1688\u001b[0m         kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(kwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/pipeline/pipe.pyx:55\u001b[0m, in \u001b[0;36mpipe\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1685\u001b[0m, in \u001b[0;36m_pipe\u001b[0;34m(docs, proc, name, default_error_handler, kwargs)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_pipe\u001b[39m(\n\u001b[1;32m   1676\u001b[0m     docs: Iterable[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   1677\u001b[0m     proc: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeCallable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1682\u001b[0m     kwargs: Mapping[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m   1683\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m   1684\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(proc, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpipe\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m-> 1685\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m proc\u001b[38;5;241m.\u001b[39mpipe(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1686\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1687\u001b[0m         \u001b[38;5;66;03m# We added some args for pipe that __call__ doesn't expect.\u001b[39;00m\n\u001b[1;32m   1688\u001b[0m         kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(kwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/pipeline/pipe.pyx:55\u001b[0m, in \u001b[0;36mpipe\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1685\u001b[0m, in \u001b[0;36m_pipe\u001b[0;34m(docs, proc, name, default_error_handler, kwargs)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_pipe\u001b[39m(\n\u001b[1;32m   1676\u001b[0m     docs: Iterable[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   1677\u001b[0m     proc: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeCallable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1682\u001b[0m     kwargs: Mapping[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m   1683\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m   1684\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(proc, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpipe\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m-> 1685\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m proc\u001b[38;5;241m.\u001b[39mpipe(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1686\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1687\u001b[0m         \u001b[38;5;66;03m# We added some args for pipe that __call__ doesn't expect.\u001b[39;00m\n\u001b[1;32m   1688\u001b[0m         kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(kwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/pipeline/transition_parser.pyx:245\u001b[0m, in \u001b[0;36mpipe\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1632\u001b[0m, in \u001b[0;36mminibatch\u001b[0;34m(items, size)\u001b[0m\n\u001b[1;32m   1630\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m   1631\u001b[0m     batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(size_)\n\u001b[0;32m-> 1632\u001b[0m     batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitertools\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mislice\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitems\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1633\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(batch) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1634\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1685\u001b[0m, in \u001b[0;36m_pipe\u001b[0;34m(docs, proc, name, default_error_handler, kwargs)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_pipe\u001b[39m(\n\u001b[1;32m   1676\u001b[0m     docs: Iterable[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   1677\u001b[0m     proc: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeCallable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1682\u001b[0m     kwargs: Mapping[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m   1683\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m   1684\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(proc, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpipe\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m-> 1685\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m proc\u001b[38;5;241m.\u001b[39mpipe(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1686\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1687\u001b[0m         \u001b[38;5;66;03m# We added some args for pipe that __call__ doesn't expect.\u001b[39;00m\n\u001b[1;32m   1688\u001b[0m         kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(kwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/pipeline/trainable_pipe.pyx:73\u001b[0m, in \u001b[0;36mpipe\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1632\u001b[0m, in \u001b[0;36mminibatch\u001b[0;34m(items, size)\u001b[0m\n\u001b[1;32m   1630\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m   1631\u001b[0m     batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(size_)\n\u001b[0;32m-> 1632\u001b[0m     batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mitertools\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mislice\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitems\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mint\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1633\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(batch) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1634\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy/util.py:1685\u001b[0m, in \u001b[0;36m_pipe\u001b[0;34m(docs, proc, name, default_error_handler, kwargs)\u001b[0m\n\u001b[1;32m   1675\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_pipe\u001b[39m(\n\u001b[1;32m   1676\u001b[0m     docs: Iterable[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   1677\u001b[0m     proc: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeCallable\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1682\u001b[0m     kwargs: Mapping[\u001b[38;5;28mstr\u001b[39m, Any],\n\u001b[1;32m   1683\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Iterator[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDoc\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m   1684\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(proc, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpipe\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m-> 1685\u001b[0m         \u001b[38;5;28;01myield from\u001b[39;00m proc\u001b[38;5;241m.\u001b[39mpipe(docs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1686\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1687\u001b[0m         \u001b[38;5;66;03m# We added some args for pipe that __call__ doesn't expect.\u001b[39;00m\n\u001b[1;32m   1688\u001b[0m         kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(kwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy_curated_transformers/pipeline/transformer.py:208\u001b[0m, in \u001b[0;36mCuratedTransformer.pipe\u001b[0;34m(self, stream, batch_size)\u001b[0m\n\u001b[1;32m    206\u001b[0m _install_extensions()\n\u001b[1;32m    207\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m minibatch(stream, batch_size):\n\u001b[0;32m--> 208\u001b[0m     preds \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    209\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mset_annotations(batch, preds)\n\u001b[1;32m    210\u001b[0m     \u001b[38;5;28;01myield from\u001b[39;00m batch\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy_curated_transformers/pipeline/transformer.py:240\u001b[0m, in \u001b[0;36mCuratedTransformer.predict\u001b[0;34m(self, docs)\u001b[0m\n\u001b[1;32m    238\u001b[0m \u001b[38;5;66;03m# To ensure that the model's internal state is always consistent with the pipe's.\u001b[39;00m\n\u001b[1;32m    239\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_set_model_all_layer_outputs(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mall_layer_outputs)\n\u001b[0;32m--> 240\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/model.py:334\u001b[0m, in \u001b[0;36mModel.predict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    330\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpredict\u001b[39m(\u001b[38;5;28mself\u001b[39m, X: InT) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m OutT:\n\u001b[1;32m    331\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Call the model's `forward` function with `is_train=False`, and return\u001b[39;00m\n\u001b[1;32m    332\u001b[0m \u001b[38;5;124;03m    only the output, instead of the `(output, callback)` tuple.\u001b[39;00m\n\u001b[1;32m    333\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 334\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m]\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy_curated_transformers/models/architectures.py:652\u001b[0m, in \u001b[0;36mtransformer_model_forward\u001b[0;34m(model, docs, is_train)\u001b[0m\n\u001b[1;32m    649\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtransformer_model_forward\u001b[39m(\n\u001b[1;32m    650\u001b[0m     model: TransformerModelT, docs: TransformerInT, is_train: \u001b[38;5;28mbool\u001b[39m\n\u001b[1;32m    651\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[TransformerOutT, TransformerBackpropT]:\n\u001b[0;32m--> 652\u001b[0m     Y, backprop_layer \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayers\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    654\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mbackprop\u001b[39m(dY):\n\u001b[1;32m    655\u001b[0m         backprop_layer(dY)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/model.py:310\u001b[0m, in \u001b[0;36mModel.__call__\u001b[0;34m(self, X, is_train)\u001b[0m\n\u001b[1;32m    307\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, X: InT, is_train: \u001b[38;5;28mbool\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[OutT, Callable]:\n\u001b[1;32m    308\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Call the model's `forward` function, returning the output and a\u001b[39;00m\n\u001b[1;32m    309\u001b[0m \u001b[38;5;124;03m    callback to compute the gradients via backpropagation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 310\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy_curated_transformers/models/with_non_ws_tokens.py:72\u001b[0m, in \u001b[0;36mwith_non_ws_tokens_forward\u001b[0;34m(model, X, is_train)\u001b[0m\n\u001b[1;32m     70\u001b[0m inner: Model[Tok2PiecesInT, WsTokenAdapterOutT] \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mlayers[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m     71\u001b[0m tokens, ws_counts \u001b[38;5;241m=\u001b[39m _filter_tokens(X)\n\u001b[0;32m---> 72\u001b[0m Y_no_ws, backprop_no_ws \u001b[38;5;241m=\u001b[39m \u001b[43minner\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtokens\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     74\u001b[0m \u001b[38;5;66;03m# Note: we modify the model outputs in-place. Since we are wrapping the\u001b[39;00m\n\u001b[1;32m     75\u001b[0m \u001b[38;5;66;03m# model, there should be no other consumers. Not sure yet if the same\u001b[39;00m\n\u001b[1;32m     76\u001b[0m \u001b[38;5;66;03m# applies to the gradient (e.g. consider summing two encoders of the\u001b[39;00m\n\u001b[1;32m     77\u001b[0m \u001b[38;5;66;03m# same width downstream.)\u001b[39;00m\n\u001b[1;32m     79\u001b[0m alignments \u001b[38;5;241m=\u001b[39m _create_alignments(model, Y_no_ws, ws_counts)\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/model.py:310\u001b[0m, in \u001b[0;36mModel.__call__\u001b[0;34m(self, X, is_train)\u001b[0m\n\u001b[1;32m    307\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, X: InT, is_train: \u001b[38;5;28mbool\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[OutT, Callable]:\n\u001b[1;32m    308\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Call the model's `forward` function, returning the output and a\u001b[39;00m\n\u001b[1;32m    309\u001b[0m \u001b[38;5;124;03m    callback to compute the gradients via backpropagation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 310\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/layers/chain.py:54\u001b[0m, in \u001b[0;36mforward\u001b[0;34m(model, X, is_train)\u001b[0m\n\u001b[1;32m     52\u001b[0m callbacks \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     53\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m model\u001b[38;5;241m.\u001b[39mlayers:\n\u001b[0;32m---> 54\u001b[0m     Y, inc_layer_grad \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     55\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mappend(inc_layer_grad)\n\u001b[1;32m     56\u001b[0m     X \u001b[38;5;241m=\u001b[39m Y\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/model.py:310\u001b[0m, in \u001b[0;36mModel.__call__\u001b[0;34m(self, X, is_train)\u001b[0m\n\u001b[1;32m    307\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, X: InT, is_train: \u001b[38;5;28mbool\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[OutT, Callable]:\n\u001b[1;32m    308\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Call the model's `forward` function, returning the output and a\u001b[39;00m\n\u001b[1;32m    309\u001b[0m \u001b[38;5;124;03m    callback to compute the gradients via backpropagation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 310\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/spacy_curated_transformers/models/with_strided_spans.py:109\u001b[0m, in \u001b[0;36mwith_strided_spans_forward\u001b[0;34m(model, X, is_train)\u001b[0m\n\u001b[1;32m    107\u001b[0m outputs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    108\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch \u001b[38;5;129;01min\u001b[39;00m _split_spans(spans, batch_size):\n\u001b[0;32m--> 109\u001b[0m     output, bp \u001b[38;5;241m=\u001b[39m \u001b[43mtransformer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcast\u001b[49m\u001b[43m(\u001b[49m\u001b[43mTorchTransformerInT\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    110\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, TransformerModelOutput):\n\u001b[1;32m    111\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    112\u001b[0m             Errors\u001b[38;5;241m.\u001b[39mE014\u001b[38;5;241m.\u001b[39mformat(model_name\u001b[38;5;241m=\u001b[39mmodel\u001b[38;5;241m.\u001b[39mname, input_type\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mtype\u001b[39m(output))\n\u001b[1;32m    113\u001b[0m         )\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/model.py:310\u001b[0m, in \u001b[0;36mModel.__call__\u001b[0;34m(self, X, is_train)\u001b[0m\n\u001b[1;32m    307\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, X: InT, is_train: \u001b[38;5;28mbool\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[OutT, Callable]:\n\u001b[1;32m    308\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Call the model's `forward` function, returning the output and a\u001b[39;00m\n\u001b[1;32m    309\u001b[0m \u001b[38;5;124;03m    callback to compute the gradients via backpropagation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 310\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/layers/pytorchwrapper.py:225\u001b[0m, in \u001b[0;36mforward\u001b[0;34m(model, X, is_train)\u001b[0m\n\u001b[1;32m    222\u001b[0m convert_outputs \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconvert_outputs\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    224\u001b[0m Xtorch, get_dX \u001b[38;5;241m=\u001b[39m convert_inputs(model, X, is_train)\n\u001b[0;32m--> 225\u001b[0m Ytorch, torch_backprop \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshims\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m(\u001b[49m\u001b[43mXtorch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_train\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    226\u001b[0m Y, get_dYtorch \u001b[38;5;241m=\u001b[39m convert_outputs(model, (X, Ytorch), is_train)\n\u001b[1;32m    228\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mbackprop\u001b[39m(dY: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/shims/pytorch.py:97\u001b[0m, in \u001b[0;36mPyTorchShim.__call__\u001b[0;34m(self, inputs, is_train)\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbegin_update(inputs)\n\u001b[1;32m     96\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 97\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;28;01mlambda\u001b[39;00m a: \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/thinc/shims/pytorch.py:115\u001b[0m, in \u001b[0;36mPyTorchShim.predict\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[1;32m    114\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mamp\u001b[38;5;241m.\u001b[39mautocast(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mixed_precision):\n\u001b[0;32m--> 115\u001b[0m         outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    116\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_model\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m    117\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m outputs\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/curated_transformers/models/curated_transformer.py:37\u001b[0m, in \u001b[0;36mCuratedTransformer.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids)\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[1;32m     28\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m     29\u001b[0m     input_ids: Tensor,\n\u001b[1;32m     30\u001b[0m     attention_mask: Optional[AttentionMask] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m     31\u001b[0m     token_type_ids: Optional[Tensor] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m     32\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m PyTorchTransformerOutput:\n\u001b[1;32m     33\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m     34\u001b[0m \u001b[38;5;124;03m    Shapes:\u001b[39;00m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;124;03m        input_ids, attention_mask, token_type_ids - (batch, seq_len)\u001b[39;00m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcurated_encoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken_type_ids\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/curated_transformers/models/roberta/encoder.py:51\u001b[0m, in \u001b[0;36mRobertaEncoder.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids)\u001b[0m\n\u001b[1;32m     49\u001b[0m layer_outputs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers:\n\u001b[0;32m---> 51\u001b[0m     layer_output \u001b[38;5;241m=\u001b[39m \u001b[43mlayer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlayer_output\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     52\u001b[0m     layer_outputs\u001b[38;5;241m.\u001b[39mappend(layer_output)\n\u001b[1;32m     54\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m PyTorchTransformerOutput(\n\u001b[1;32m     55\u001b[0m     embedding_output\u001b[38;5;241m=\u001b[39membeddings, layer_hidden_states\u001b[38;5;241m=\u001b[39mlayer_outputs\n\u001b[1;32m     56\u001b[0m )\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/curated_transformers/models/bert/layer.py:135\u001b[0m, in \u001b[0;36mBertEncoderLayer.forward\u001b[0;34m(self, x, attn_mask)\u001b[0m\n\u001b[1;32m    133\u001b[0m ffn_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mffn(attn_out)\n\u001b[1;32m    134\u001b[0m ffn_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mffn_output_dropout(ffn_out)\n\u001b[0;32m--> 135\u001b[0m ffn_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mffn_output_layernorm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mattn_out\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mffn_out\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ffn_out\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/modules/normalization.py:196\u001b[0m, in \u001b[0;36mLayerNorm.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 196\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    197\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnormalized_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meps\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/exp/lib/python3.10/site-packages/torch/nn/functional.py:2543\u001b[0m, in \u001b[0;36mlayer_norm\u001b[0;34m(input, normalized_shape, weight, bias, eps)\u001b[0m\n\u001b[1;32m   2539\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_variadic(\u001b[38;5;28minput\u001b[39m, weight, bias):\n\u001b[1;32m   2540\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m   2541\u001b[0m         layer_norm, (\u001b[38;5;28minput\u001b[39m, weight, bias), \u001b[38;5;28minput\u001b[39m, normalized_shape, weight\u001b[38;5;241m=\u001b[39mweight, bias\u001b[38;5;241m=\u001b[39mbias, eps\u001b[38;5;241m=\u001b[39meps\n\u001b[1;32m   2542\u001b[0m     )\n\u001b[0;32m-> 2543\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlayer_norm\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnormalized_shape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meps\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackends\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcudnn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menabled\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 2000\n",
    "texts = df[df.answer_len < 5].answer.tolist() + df[df.answer_len < 5].question.tolist()\n",
    "docs = []\n",
    "for i in tqdm(range(0, len(texts), BATCH_SIZE)):\n",
    "    batch = texts[i:i+BATCH_SIZE]\n",
    "    docs.extend(list(ner.pipe(batch)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in docs:\n",
    "    for ent in doc.ents:\n",
    "        ent2text[ent.label_].append(ent.text)\n",
    "    for token in doc:\n",
    "        if not token.ent_type_:\n",
    "            pos2text[token.pos_].append(token.text)\n",
    "for k, v in ent2text.items():\n",
    "    ent2text[k] = list(set(v))\n",
    "for k, v in ent2text.items():\n",
    "    for vv in v:\n",
    "        text2ent[vv] = k\n",
    "pos2text = {k:list(set(v)) for k, v in pos2text.items() if k in ANSWER_POS}\n",
    "for k, v in pos2text.items():\n",
    "    for vv in v:\n",
    "        text2pos[vv] = k\n",
    "print(len(text2ent), len(text2pos), len(ent2text), len(pos2text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b71764c9d574833ae886704e654f86a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/18 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e7bb00c6d21466aa661682ec2cc513b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "\n",
    "ent2text_vec = dict()\n",
    "for k, v in tqdm(ent2text.items()):\n",
    "    docs  = nlp.pipe(v)\n",
    "    ent2text_vec[k] = [doc.vector for doc in docs]\n",
    "docs = nlp.pipe(list(text2ent.keys()))\n",
    "text2ent_vec = dict()\n",
    "for doc in docs:\n",
    "    text2ent_vec[doc.text] = doc.vector\n",
    "pos2text_vec = dict()\n",
    "for k, v in tqdm(pos2text.items()):\n",
    "    docs  = nlp.pipe(v)\n",
    "    pos2text_vec[k] = [doc.vector for doc in docs]\n",
    "for k, v in pos2text.items():\n",
    "    for vv in v:\n",
    "        text2pos[vv] = k\n",
    "text2pos_vec = dict()\n",
    "docs = nlp.pipe(list(text2pos.keys()))\n",
    "for doc in docs:\n",
    "    text2pos_vec[doc.text] = doc.vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18 95515 6 50872\n",
      "18 95515 6 50872\n"
     ]
    }
   ],
   "source": [
    "print(len(ent2text), len(text2ent), len(pos2text), len(text2pos))\n",
    "print(len(ent2text_vec), len(text2ent_vec), len(pos2text_vec), len(text2pos_vec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9ddb8dac63345e7abe0e7c580fb07dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ent2text_vec:   0%|          | 0/18 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2596079/2671793966.py:6: RuntimeWarning: divide by zero encountered in divide\n",
      "  normalized.append(numpy_vector / np.linalg.norm(numpy_vector))\n",
      "/tmp/ipykernel_2596079/2671793966.py:6: RuntimeWarning: invalid value encountered in divide\n",
      "  normalized.append(numpy_vector / np.linalg.norm(numpy_vector))\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa86472433674cd799615e2a5c907814",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pos2text_vec:   0%|          | 0/6 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2596079/2671793966.py:12: RuntimeWarning: divide by zero encountered in divide\n",
      "  normalized.append(numpy_vector / np.linalg.norm(numpy_vector))\n",
      "/tmp/ipykernel_2596079/2671793966.py:12: RuntimeWarning: invalid value encountered in divide\n",
      "  normalized.append(numpy_vector / np.linalg.norm(numpy_vector))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/data/seongil/datasets/text2pos_vec.joblib']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "for k, v in tqdm(ent2text_vec.items(), total=len(ent2text_vec), desc=\"ent2text_vec\"):\n",
    "    normalized = []\n",
    "    for vv in v:\n",
    "        numpy_vector = vv.get()\n",
    "        normalized.append(numpy_vector / np.linalg.norm(numpy_vector))\n",
    "    ent2text_vec[k] = np.array(normalized)\n",
    "for k, v in tqdm(pos2text_vec.items(), total=len(pos2text_vec), desc=\"pos2text_vec\"):\n",
    "    normalized = []\n",
    "    for vv in v:\n",
    "        numpy_vector = vv.get()\n",
    "        normalized.append(numpy_vector / np.linalg.norm(numpy_vector))\n",
    "    pos2text_vec[k] = np.array(normalized)\n",
    "joblib.dump(ent2text_vec, \"/data/seongil/datasets/ent2text_vec.joblib\")\n",
    "joblib.dump(text2ent_vec, \"/data/seongil/datasets/text2ent_vec.joblib\")\n",
    "joblib.dump(pos2text_vec, \"/data/seongil/datasets/pos2text_vec.joblib\")\n",
    "joblib.dump(text2pos_vec, \"/data/seongil/datasets/text2pos_vec.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/data/seongil/datasets/text2pos.joblib']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(ent2text, \"/data/seongil/datasets/ent2text.joblib\")\n",
    "joblib.dump(text2ent, \"/data/seongil/datasets/text2ent.joblib\")\n",
    "joblib.dump(pos2text, \"/data/seongil/datasets/pos2text.joblib\")\n",
    "joblib.dump(text2pos, \"/data/seongil/datasets/text2pos.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "ent2text_vec = joblib.load(\"/data/seongil/datasets/ent2text_vec.joblib\")\n",
    "text2ent_vec = joblib.load(\"/data/seongil/datasets/text2ent_vec.joblib\")\n",
    "pos2text_vec = joblib.load(\"/data/seongil/datasets/pos2text_vec.joblib\")\n",
    "text2pos_vec = joblib.load(\"/data/seongil/datasets/text2pos_vec.joblib\")\n",
    "ent2text = joblib.load(\"/data/seongil/datasets/ent2text.joblib\")\n",
    "text2ent = joblib.load(\"/data/seongil/datasets/text2ent.joblib\")\n",
    "pos2text = joblib.load(\"/data/seongil/datasets/pos2text.joblib\")\n",
    "text2pos = joblib.load(\"/data/seongil/datasets/text2pos.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34150"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos2text[\"NOUN\"].index(\"state.[citation\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos2text[\"NOUN\"] = [item for idx, item in enumerate(pos2text[\"NOUN\"]) if idx != 34150]\n",
    "pos2text_vec[\"NOUN\"] = [item for idx, item in enumerate(pos2text_vec[\"NOUN\"]) if idx != 34150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos2text_vec[\"NOUN\"] = cp.array(pos2text_vec[\"NOUN\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'ndarray' object has no attribute 'pop'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[103], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m pos2text[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNOUN\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;241m34150\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m \u001b[43mpos2text_vec\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mNOUN\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m(\u001b[38;5;241m34150\u001b[39m)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'ndarray' object has no attribute 'pop'"
     ]
    }
   ],
   "source": [
    "pos2text[\"NOUN\"].pop(34150)\n",
    "pos2text_vec[\"NOUN\"].pop(34150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "list.remove(x): x not in list",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[101], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m a \u001b[38;5;241m=\u001b[39m [\u001b[38;5;241m12\u001b[39m,\u001b[38;5;241m3\u001b[39m]\n\u001b[0;32m----> 2\u001b[0m \u001b[43ma\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mremove\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mValueError\u001b[0m: list.remove(x): x not in list"
     ]
    }
   ],
   "source": [
    "a = [12,3]\n",
    "a.remove(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'state.[citation'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[96], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtext2pos\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstate.[citation\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'state.[citation'"
     ]
    }
   ],
   "source": [
    "text2pos[\"state.[citation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "from gensim.test.utils import datapath\n",
    "wv = KeyedVectors.load_word2vec_format(datapath('/data/seongil/wiki/enwiki_20180420_100d.txt'), binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ENTITY/Super_Bowl_VIII', 0.813761830329895),\n",
       " ('ENTITY/Super_Bowl_XXIV', 0.7884112596511841),\n",
       " ('ENTITY/AFC_Championship_Game', 0.7823604345321655),\n",
       " ('ENTITY/Super_Bowl_XVII', 0.7784186601638794),\n",
       " ('ENTITY/Super_Bowl_XVIII', 0.7771480083465576),\n",
       " ('ENTITY/Super_Bowl_Most_Valuable_Player_Award', 0.7717746496200562),\n",
       " ('ENTITY/National_Football_Conference', 0.7650311589241028),\n",
       " ('ENTITY/Super_Bowl_XXVI', 0.7605806589126587),\n",
       " ('ENTITY/2005_Pittsburgh_Steelers_season', 0.7526757717132568),\n",
       " ('ENTITY/Super_Bowl_IV', 0.7526056170463562)]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.similar_by_word(\"ENTITY/Super_Bowl\", topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv.cosine_similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'KeyedVectors' object has no attribute 'similarities'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[114], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mwv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimilarities\u001b[49m([\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mENTITY/Super_Bowl\u001b[39m\u001b[38;5;124m\"\u001b[39m], [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mENTITY/Super_Bowl\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mENTITY/Super_Bowl_Most_Valuable_Player_Award\u001b[39m\u001b[38;5;124m'\u001b[39m,])\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'KeyedVectors' object has no attribute 'similarities'"
     ]
    }
   ],
   "source": [
    "wv.similarities([\"ENTITY/Super_Bowl\"], [\"ENTITY/Super_Bowl\",'ENTITY/Super_Bowl_Most_Valuable_Player_Award',])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "ner = spacy.load(\"en_core_web_trf\")\n",
    "emb = spacy.load(\"en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim.downloader as api\n",
    "model = api.load(\"glove-wiki-gigaword-300\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as cp\n",
    "import pandas as pd\n",
    "import random\n",
    "import numpy as np\n",
    "import cupy as cp\n",
    "from numpy.linalg import norm\n",
    "EDIT_CANDIDATE_POS = [\"VERB\", \"NOUN\", \"ADJ\", \"ADV\"]\n",
    "ANSWER_POS = [\"ADV\", \"ADJ\", \"NOUN\", \"VERB\", \"NUM\"]\n",
    "POS_ALIGN = [\"VERB\", \"NOUN\", \"ADJ\"]\n",
    "from nltk.corpus import wordnet as wn\n",
    "import random\n",
    "def _sample_token_same_level_based_on_pos(token, pos):\n",
    "    possible_candidates = []\n",
    "    synsets = wn.synsets(token)\n",
    "    if len(synsets) == 0:\n",
    "        return possible_candidates\n",
    "    same_synsets = []\n",
    "    for synset in synsets:\n",
    "        if synset.lemma_names()[0] == token and synset.lexname().split(\".\")[0] == pos:\n",
    "            same_synsets.append(synset)\n",
    "    possible_candidates = []\n",
    "    for synset in same_synsets:\n",
    "        hypernyms = synset.hypernyms()\n",
    "        for hyper in hypernyms:\n",
    "            hypos = hyper.hyponyms()\n",
    "            hypos = [h.lemma_names()[0].replace(\"_\", \" \") for h in hypos if h.lemma_names()[0] != token]\n",
    "            possible_candidates.extend(hypos)\n",
    "    return possible_candidates\n",
    "def find_most_similar(word, word_list,top_n=1):\n",
    "    \"\"\"\n",
    "    주어진 단어와 단어 리스트에서 가장 유사한 top_n 개의 단어들을 찾는 함수.\n",
    "\n",
    "    :param word: 유사성을 찾을 단어\n",
    "    :param word_list: 유사성을 비교할 단어 리스트\n",
    "    :param model: GloVe 모델\n",
    "    :param top_n: 반환할 상위 단어의 수\n",
    "    :return: 가장 유사한 top_n 개의 단어 리스트\n",
    "    \"\"\"\n",
    "    # 모델에 없는 단어를 필터링\n",
    "    word_list = [w for w in word_list if w in model.key_to_index]\n",
    "    if word_list == []:\n",
    "        return None\n",
    "    # 각 단어에 대한 유사도 계산\n",
    "    similarity_list = [(w, model.similarity(word, w)) for w in word_list]\n",
    "\n",
    "    # 유사도에 따라 정렬\n",
    "    sorted_similarity = sorted(similarity_list, key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    # 상위 n개의 단어 반환\n",
    "    return sorted_similarity[0][0] if sorted_similarity[0][1] > 0.6 else None\n",
    "\n",
    "def check_cosine_similarity(arr1, arr2, thres):\n",
    "    cosine_similarity = cp.dot(arr1, arr2) / (norm(arr1) * norm(arr2))\n",
    "    return (cosine_similarity <= 0.95) and (cosine_similarity >= thres)\n",
    "    \n",
    "def find_non_identical_random_entity(ent_type, origin: str, thres: float=0.9):\n",
    "    subsets = ent2text_vec[ent_type]\n",
    "    #random_entities = random.sample(subsets, 100 if len(subsets)>100 else len(subsets))\n",
    "    for entity in subsets:#random_entities:\n",
    "        origin_vec, entity_vec = text2ent_vec[origin], text2ent_vec[entity]\n",
    "        for thres in [0.8]:\n",
    "            if check_cosine_similarity(origin_vec, entity_vec, thres):\n",
    "                return entity\n",
    "    return None\n",
    "\n",
    "def find_random_entity(ent_type, origin: str):\n",
    "    subsets = ent2text_vec[ent_type]\n",
    "    random_entity = random.choice(subsets)\n",
    "    while random_entity == origin:\n",
    "        random_entity = random.choice(subsets)\n",
    "    return random_entity\n",
    "\n",
    "def find_similar_word(token: str, pos: str):\n",
    "    candidates = _sample_token_same_level_based_on_pos(token, pos)\n",
    "    filtered_candidates = find_most_similar(token, candidates)\n",
    "    if filtered_candidates:\n",
    "        return filtered_candidates\n",
    "    else:\n",
    "        return None\n",
    "        \n",
    "def find_topk(query, key, topk: int):\n",
    "    query = query / cp.linalg.norm(query)\n",
    "    output = cp.dot(key, query.T)\n",
    "    output_idx = []\n",
    "    for val in np.argpartition(output, -topk)[-topk:]:\n",
    "        output_idx.append(int(val))\n",
    "    return output_idx\n",
    "\n",
    "def find_similar_entity(entity_type: str, answer: str):\n",
    "    result = \"\"\n",
    "    try:\n",
    "        topn = wv.similar_by_word(f\"ENTITY/{answer.replace(' ', '_')}\", topn=10)\n",
    "        for word, similarity in topn:\n",
    "            if similarity > 0.8:\n",
    "                continue\n",
    "            elif answer in word:\n",
    "                continue\n",
    "            else:\n",
    "                result = word.split(\"/\")[-1].replace(\"_\", \" \")\n",
    "                break\n",
    "    except:\n",
    "        topn = find_topk(text2ent_vec[answer], ent2text_vec[entity_type], 10)\n",
    "        for idx in topn:\n",
    "            if ent2text[entity_type][idx] == answer:\n",
    "                continue\n",
    "            else:\n",
    "                result = ent2text[entity_type][idx]\n",
    "                break\n",
    "    print(\"Result: \", result)\n",
    "    return result\n",
    "    \n",
    "def make_adversarial_sentence(answer, answer_sentence, question_doc, answer_sentence_doc):\n",
    "    keywords = set()\n",
    "    keywords.add(answer)\n",
    "    for token in question_doc:\n",
    "        keywords.add(token.lemma_)\n",
    "    for ent in question_doc.ents:\n",
    "        keywords.add(ent.text)\n",
    "    keywords = list(keywords)\n",
    "    \n",
    "    replaced_entity = dict()\n",
    "    for ent in answer_sentence_doc.ents:\n",
    "        if ent and (ent.text not in keywords) and (ent.text not in replaced_entity):\n",
    "            replaced_entity[ent.text] = find_similar_entity(ent.label_, ent.text)\n",
    "    replaced_words = dict()\n",
    "    for token in answer_sentence_doc:\n",
    "        if (token.lemma_ not in keywords) and (not token.ent_type_) and (token.text not in replaced_words):\n",
    "            if token.pos_ == 'NUM':\n",
    "                replaced_words[token.text] = random.choice(pos2text['NUM'])\n",
    "            else:\n",
    "                replaced_words[token.text] = find_similar_word(token.lemma_.lower(), token.pos_.lower())\n",
    "    \n",
    "    for before, after in replaced_entity.items():\n",
    "        if not after:\n",
    "            continue\n",
    "        print(\"Before: \", before, \"After: \", after)\n",
    "        answer_sentence = answer_sentence.replace(before, after)\n",
    "    for before, after in replaced_words.items():\n",
    "        if not after:\n",
    "            continue\n",
    "        print(\"Before: \", before, \"After: \", after)\n",
    "        answer_sentence = answer_sentence.replace(before, after)\n",
    "    \n",
    "    ### Answer를 바꿔주는 작업!\n",
    "    if answer in text2ent:\n",
    "        answer_sentence = answer_sentence.replace(answer, find_similar_entity(text2ent[answer], answer))\n",
    "    else:\n",
    "        answer_sentence = answer_sentence.replace(answer, find_similar_word(text2pos[answer], answer))\n",
    "    return answer_sentence\n",
    "\n",
    "def determine_answer_type(answer):\n",
    "    if len(answer.split()) == 1:\n",
    "        return \"word\"\n",
    "    elif len(answer.split()) < 5:\n",
    "        return \"phrase\"\n",
    "    else:\n",
    "        return \"sentence\"\n",
    "\n",
    "def preprocess_answer(answer: str):\n",
    "    ans_type = determine_answer_type(answer)\n",
    "    if ans_type == \"word\":\n",
    "        return answer\n",
    "    elif ans_type == \"phrase\":\n",
    "        return answer\n",
    "    elif ans_type == \"sentence\":\n",
    "        doc = ner(answer)\n",
    "        return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27064154d98f4ff4803bcbd7870ea72d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "second||first||Tsang||Karma Phuntsok Namgyal||Tibet||Ü-Tsang||Result:  fourth\n",
      "Result:  second\n",
      "Result:  Berkeleian\n",
      "Result:  \n",
      "Before:  second After:  fourth\n",
      "Before:  first After:  second\n",
      "Before:  Tsang After:  Berkeleian\n",
      "Before:  king After:  queen\n",
      "Before:  1611–1621 After:  232\n",
      "Question: Who took control of the whole of Central Tibet?\n",
      "Answer: Karma Phuntsok Namgyal\n",
      "Answer Sent: The second successor of this first Tsang king, Karma Phuntsok Namgyal, took control of the whole of Central Tibet (Ü-Tsang), reigning from 1611–1621.\n",
      "New Answer Sent: The fourth successor of this second Berkeleian queen, Karma Phuntsok Namgyal, took control of the whole of Central Tibet (Ü-Berkeleian), reigning from 232.\n",
      "--------------------------------------------------\n",
      "Before:  monastery After:  convent\n",
      "Question: Customarily where is the expected place in the lives of the people for the home of monks in Burma ?\n",
      "Answer: In a traditional village, the monastery is the centre of cultural life.\n",
      "Answer Sent: In a traditional village, the monastery is the centre of cultural life.\n",
      "New Answer Sent: In a traditional village, the convent is the centre of cultural life.\n",
      "--------------------------------------------------\n",
      "Jews||Ashkenazi||Haredi Jews||Hasidic||Hareidi||Ashkenazi Jews||Result:  jewish\n",
      "Result:  Berkeleian\n",
      "Result:  Berkeleian\n",
      "Result:  Berkeleian\n",
      "Result:  Afrikaner\n",
      "Result:  sephardim\n",
      "Before:  Jews After:  jewish\n",
      "Before:  Ashkenazi After:  Berkeleian\n",
      "Before:  Haredi Jews After:  Berkeleian\n",
      "Before:  Hasidic After:  Berkeleian\n",
      "Before:  Hareidi After:  Afrikaner\n",
      "Before:  Ashkenazi Jews After:  sephardim\n",
      "Question: Members of which two Haredi Jewish sects continue to marry exclusively within the Ashkenazi Jewish population?\n",
      "Answer: Hasidic or Hareidi sects\n",
      "Answer Sent: Moreover, though intermarriage among Jews of Ashkenazi descent has become increasingly common, many Haredi Jews, particularly members of Hasidic or Hareidi sects, continue to marry exclusively fellow Ashkenazi Jews.\n",
      "New Answer Sent: Moreover, though intermarriage among jewish of Berkeleian descent has become increasingly common, many Haredi jewish, particularly members of Berkeleian or Afrikaner sects, continue to marry exclusively fellow Berkeleian jewish.\n",
      "--------------------------------------------------\n",
      "Sears, Roebuck & Company||first||eight||1925||338||1930||595||1940||Result:  MARLANT\n",
      "Result:  second\n",
      "Result:  6,600,299\n",
      "Result:  1921\n",
      "Result:  6,600,299\n",
      "Result:  1928\n",
      "Before:  Sears, Roebuck & Company After:  MARLANT\n",
      "Before:  first After:  second\n",
      "Before:  eight After:  6,600,299\n",
      "Before:  1925 After:  1921\n",
      "Before:  338 After:  6,600,299\n",
      "Before:  1930 After:  1928\n",
      "Before:  house After:  senate\n",
      "Question: How many stores was Sears operating in 1940?\n",
      "Answer: 595\n",
      "Answer Sent: Sears, Roebuck & Company, a giant mail-order house, opened its first eight retail stores in 1925, and operated 338 by 1930, and 595 by 1940.\n",
      "New Answer Sent: MARLANT, a giant mail-order senate, opened its second 6,600,299 retail stores in 1921, and operated 6,600,299 by 1928, and 595 by 1940.\n",
      "--------------------------------------------------\n",
      "two||Question: What two groups are annelids traditionally divided among?\n",
      "Answer: polychaetes and clitellates\n",
      "Answer Sent: Traditionally the annelids have been divided into two major groups, the polychaetes and clitellates.\n",
      "New Answer Sent: Traditionally the annelids have been divided into two major groups, the polychaetes and clitellates.\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "## Answer이 Phrase인 경우\n",
    "## Answer이 Sentence인 경우\n",
    "\n",
    "for i, row in tqdm(df.sample(5).iterrows()):\n",
    "    question, answer, answer_sent = row[\"question\"], row['answer'], row['answer_sent']\n",
    "    question_doc, answer_sent_doc = ner(question), ner(answer_sent)\n",
    "    for ent in answer_sent_doc.ents:\n",
    "        print(ent.text, end=\"||\")\n",
    "    new_answer_sent = make_adversarial_sentence(answer, answer_sent, question_doc, answer_sent_doc)\n",
    "    print(\"Question:\", question)\n",
    "    print(\"Answer:\", answer)\n",
    "    print(\"Answer Sent:\", answer_sent)\n",
    "    print(\"New Answer Sent:\", new_answer_sent)\n",
    "    print(\"-\"*50)\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /home/seongilpark/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Republic_of_Ecuador']"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import wordnet as wn\n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "def find_synonyms_excluding_aliases(word):\n",
    "    \"\"\"\n",
    "    Finds synonyms of a given word using WordNet, excluding aliases.\n",
    "\n",
    "    :param word: The word for which synonyms are to be found.\n",
    "    :return: A set of synonyms excluding aliases.\n",
    "    \"\"\"\n",
    "    synonyms = set()\n",
    "    # Iterate over all synsets of the word\n",
    "    for synset in wn.synsets(word):\n",
    "        # Check each lemma in the synset\n",
    "        for lemma in synset.lemmas():\n",
    "            # Exclude if the lemma name is the same as the word (alias)\n",
    "            if lemma.name().lower() != word.lower():\n",
    "                synonyms.add(lemma.name())\n",
    "\n",
    "    return list(synonyms)\n",
    "\n",
    "# Example usage\n",
    "word = \"Ecuador\"\n",
    "synonyms = find_synonyms_excluding_aliases(word)\n",
    "synonyms\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in tqdm(df.sample(10).iterrows()):\n",
    "    print(\"Context: \", row['context'])\n",
    "    print(f\"Question: {row['question']}\")\n",
    "    print(f\"Correct Answer: {row['answers']['text'][0]}\")\n",
    "    print(f\"Answer Sentence: {row['answer_sent']}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'love.v.01'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.synsets(\"loved\")[0].name()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: -c: line 1: syntax error near unexpected token `'wordnet''\n",
      "/bin/bash: -c: line 1: `nltk.download('wordnet')'\n"
     ]
    }
   ],
   "source": [
    "!nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answers</th>\n",
       "      <th>masked_query</th>\n",
       "      <th>query_embedding</th>\n",
       "      <th>answer_cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5733be284776f41900661182</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>{'answer_start': [515], 'text': ['Saint Bernad...</td>\n",
       "      <td>To whom did the Virgin Mary allegedly appear i...</td>\n",
       "      <td>[-0.04606574401259422, 0.015957484021782875, -...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5733be284776f4190066117f</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is in front of the Notre Dame Main Building?</td>\n",
       "      <td>{'answer_start': [188], 'text': ['a copper sta...</td>\n",
       "      <td>What is in front of [MASK]?</td>\n",
       "      <td>[0.19923186302185059, 0.06610594689846039, 0.2...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5733be284776f41900661180</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>The Basilica of the Sacred heart at Notre Dame...</td>\n",
       "      <td>{'answer_start': [279], 'text': ['the Main Bui...</td>\n",
       "      <td>[MASK] at [MASK] is beside to which structure?</td>\n",
       "      <td>[0.3204971253871918, 0.20272424817085266, 0.22...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5733be284776f41900661181</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What is the Grotto at Notre Dame?</td>\n",
       "      <td>{'answer_start': [381], 'text': ['a Marian pla...</td>\n",
       "      <td>What is the Grotto at [MASK]?</td>\n",
       "      <td>[0.020319189876317978, 0.21503326296806335, 0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5733be284776f4190066117e</td>\n",
       "      <td>University_of_Notre_Dame</td>\n",
       "      <td>Architecturally, the school has a Catholic cha...</td>\n",
       "      <td>What sits on top of the Main Building at Notre...</td>\n",
       "      <td>{'answer_start': [92], 'text': ['a golden stat...</td>\n",
       "      <td>What sits on top of [MASK] at [MASK]?</td>\n",
       "      <td>[0.21866606175899506, 0.32595109939575195, 0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97880</th>\n",
       "      <td>5737a9afc3c5551400e51f63</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What is the exchange of heat associated with?</td>\n",
       "      <td>{'answer_start': [188, 188, 188, 188], 'text':...</td>\n",
       "      <td>What is the exchange of heat associated with?</td>\n",
       "      <td>[0.17604947090148926, 0.2812568247318268, -0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97882</th>\n",
       "      <td>5737a9afc3c5551400e51f65</td>\n",
       "      <td>Force</td>\n",
       "      <td>The connection between macroscopic nonconserva...</td>\n",
       "      <td>What makes energy changes in a closed system?</td>\n",
       "      <td>{'answer_start': [361, 361, 361, 361], 'text':...</td>\n",
       "      <td>What makes energy changes in a closed system?</td>\n",
       "      <td>[0.0338413380086422, -0.006280017085373402, 0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97884</th>\n",
       "      <td>5737aafd1c456719005744fc</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What is the kilogram-force sometimes reffered ...</td>\n",
       "      <td>{'answer_start': [114, 114, 114, 114, 114], 't...</td>\n",
       "      <td>What is the kilogram-force sometimes reffered ...</td>\n",
       "      <td>[0.13551265001296997, 0.04391534999012947, 0.1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97886</th>\n",
       "      <td>5737aafd1c456719005744fe</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What seldom used term of a unit of force equal...</td>\n",
       "      <td>{'answer_start': [712, 712, 712, 712, 712], 't...</td>\n",
       "      <td>What seldom used term of a unit of force equal...</td>\n",
       "      <td>[-0.10803636163473129, 0.0015136328293010592, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97887</th>\n",
       "      <td>5737aafd1c456719005744ff</td>\n",
       "      <td>Force</td>\n",
       "      <td>The pound-force has a metric counterpart, less...</td>\n",
       "      <td>What is the seldom used force unit equal to on...</td>\n",
       "      <td>{'answer_start': [665, 665, 665, 665, 665], 't...</td>\n",
       "      <td>What is the seldom used force unit equal to [M...</td>\n",
       "      <td>[-0.049544744193553925, -0.0007250725757330656...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92286 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             id  ... answer_cnt\n",
       "0      5733be284776f41900661182  ...          1\n",
       "1      5733be284776f4190066117f  ...          1\n",
       "2      5733be284776f41900661180  ...          1\n",
       "3      5733be284776f41900661181  ...          1\n",
       "4      5733be284776f4190066117e  ...          1\n",
       "...                         ...  ...        ...\n",
       "97880  5737a9afc3c5551400e51f63  ...          1\n",
       "97882  5737a9afc3c5551400e51f65  ...          1\n",
       "97884  5737aafd1c456719005744fc  ...          1\n",
       "97886  5737aafd1c456719005744fe  ...          1\n",
       "97887  5737aafd1c456719005744ff  ...          1\n",
       "\n",
       "[92286 rows x 8 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def make_adversary_verb_unchanged(original_sentence: str):\n",
    "    pass\n",
    "\n",
    "def make_adversary_verb_changed(original_sentence: str):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "from gensim.test.utils import datapath\n",
    "wv_from_text = KeyedVectors.load_word2vec_format(datapath('/data/seongil/wiki/enwiki_20180420_300d.txt'), binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad62f7e42fd54b53809044675ec17cf6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading readme:   0%|          | 0.00/980 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdaded4eadb74c16989d36886966ec58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5774c52b9ce4aeb8786079bd8396c3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/154M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c86706f209c48c79acab8c88f6e56b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7534143bac584ff4931d40726a0d3cb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/25866 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = pd.DataFrame(load_dataset(\"Seongill/squad_conflict_v2_under_150_with_substitution_chunked\", split=\"train\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4a9863d8c094756b639544ce0c0d16f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def find_wiki_entity(answer):\n",
    "    result = None\n",
    "    try:\n",
    "        topn = wv_from_text.similar_by_word(f\"ENTITY/{answer.replace(' ', '_')}\", topn=10)\n",
    "        for word, similarity in topn:\n",
    "            if similarity > 0.8:\n",
    "                continue\n",
    "            elif answer in word:\n",
    "                continue\n",
    "            else:\n",
    "                result = word.split(\"/\")[-1].replace(\"_\", \" \")\n",
    "                break\n",
    "    except:\n",
    "        pass\n",
    "    return result\n",
    "\n",
    "res = []\n",
    "for i, row in tqdm(data.iterrows()):\n",
    "    res.append(find_wiki_entity(row[\"answer\"]))\n",
    "data[\"wiki_similar\"] = res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"wiki_similar\"] = data.apply(lambda x: x[\"wiki_similar\"] if x[\"wiki_similar\"] != None and (x[\"wiki_similar\"].lower().strip() not in x[\"answer\"].lower().strip()) else None, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answer</th>\n",
       "      <th>similar_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21711</th>\n",
       "      <td>Ellice's Island</td>\n",
       "      <td>Pitcairn Island</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21715</th>\n",
       "      <td>Elekana</td>\n",
       "      <td>Harry Gordon Selfridge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23607</th>\n",
       "      <td>Newton</td>\n",
       "      <td>Cam Newton</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>Darlette Johnson</td>\n",
       "      <td>Thomas Sanders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7532</th>\n",
       "      <td>Red Sea</td>\n",
       "      <td>the Black Sea</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12230</th>\n",
       "      <td>Jack Brickhouse</td>\n",
       "      <td>Thomas Edward Gordon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21139</th>\n",
       "      <td>Maryland</td>\n",
       "      <td>Charleston County</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8191</th>\n",
       "      <td>Venice Park</td>\n",
       "      <td>Huntington Beach</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18067</th>\n",
       "      <td>The Syndicat des transports d'Île-de-France</td>\n",
       "      <td>Enen-kio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16380</th>\n",
       "      <td>Popes Quay</td>\n",
       "      <td>West Quay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16715</th>\n",
       "      <td>Coleman Young</td>\n",
       "      <td>Dudley Simpson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4525</th>\n",
       "      <td>Tony Curtis</td>\n",
       "      <td>Matt Lauer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10393</th>\n",
       "      <td>£9.4 billion</td>\n",
       "      <td>nearly $12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22131</th>\n",
       "      <td>$6 billion</td>\n",
       "      <td>€624 billion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9654</th>\n",
       "      <td>Oceania</td>\n",
       "      <td>Southeast Asia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25324</th>\n",
       "      <td>Doctor Who</td>\n",
       "      <td>Mahabharata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11795</th>\n",
       "      <td>the 16th century</td>\n",
       "      <td>the first Saturday and Sunday of May</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12734</th>\n",
       "      <td>Plutarco Elias Calles</td>\n",
       "      <td>Alberto Barbieri</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10853</th>\n",
       "      <td>Federal Trade Commission</td>\n",
       "      <td>Countrywide Financial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14901</th>\n",
       "      <td>Greensboro Coliseum</td>\n",
       "      <td>Wembley Arena</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            answer  \\\n",
       "21711                              Ellice's Island   \n",
       "21715                                      Elekana   \n",
       "23607                                       Newton   \n",
       "115                               Darlette Johnson   \n",
       "7532                                       Red Sea   \n",
       "12230                              Jack Brickhouse   \n",
       "21139                                     Maryland   \n",
       "8191                                   Venice Park   \n",
       "18067  The Syndicat des transports d'Île-de-France   \n",
       "16380                                   Popes Quay   \n",
       "16715                                Coleman Young   \n",
       "4525                                   Tony Curtis   \n",
       "10393                                 £9.4 billion   \n",
       "22131                                   $6 billion   \n",
       "9654                                       Oceania   \n",
       "25324                                   Doctor Who   \n",
       "11795                             the 16th century   \n",
       "12734                        Plutarco Elias Calles   \n",
       "10853                     Federal Trade Commission   \n",
       "14901                          Greensboro Coliseum   \n",
       "\n",
       "                             similar_answer  \n",
       "21711                       Pitcairn Island  \n",
       "21715                Harry Gordon Selfridge  \n",
       "23607                            Cam Newton  \n",
       "115                          Thomas Sanders  \n",
       "7532                          the Black Sea  \n",
       "12230                  Thomas Edward Gordon  \n",
       "21139                     Charleston County  \n",
       "8191                       Huntington Beach  \n",
       "18067                              Enen-kio  \n",
       "16380                             West Quay  \n",
       "16715                        Dudley Simpson  \n",
       "4525                             Matt Lauer  \n",
       "10393                            nearly $12  \n",
       "22131                          €624 billion  \n",
       "9654                         Southeast Asia  \n",
       "25324                           Mahabharata  \n",
       "11795  the first Saturday and Sunday of May  \n",
       "12734                      Alberto Barbieri  \n",
       "10853                 Countrywide Financial  \n",
       "14901                         Wembley Arena  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sample(20)[[\"answer\", \"similar_answer\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a77b0c7fcbe47b99de04a0d14d4c6ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "673c666bb3804f09b17c6f14ebc97346",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/26 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "Dataset.from_pandas(data).push_to_hub(\"Seongill/squad_conflict_v2_under_150_with_substitution_chunked_with_wiki_similar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_list = list(wv_from_text.key_to_index.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Levenshtein as lev\n",
    "\n",
    "def find_closest_word(word, word_list):\n",
    "    \"\"\"\n",
    "    Finds the closest word from the word_list to the given word using Levenshtein distance.\n",
    "    \n",
    "    :param word: The word to compare to.\n",
    "    :param word_list: A list of words to compare against.\n",
    "    :return: The closest word from the word_list.\n",
    "    \"\"\"\n",
    "    closest_word = None\n",
    "    min_distance = float('inf')\n",
    "    \n",
    "    for w in word_list:\n",
    "        distance = lev.distance(word, w)\n",
    "        if distance < min_distance:\n",
    "            min_distance = distance\n",
    "            closest_word = w\n",
    "    \n",
    "    return closest_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('seminary', 0.8390635848045349),\n",
       " ('moreau', 0.8390635848045349),\n",
       " ('billaudèle', 0.8261560797691345),\n",
       " ('dujarie', 0.8200343251228333),\n",
       " ('pététot', 0.8119063973426819),\n",
       " ('decorsant', 0.8097543120384216),\n",
       " ('katigondo', 0.8091543316841125),\n",
       " ('nyakibanda', 0.808478057384491),\n",
       " ('scholasticate', 0.8075365424156189),\n",
       " ('desproux', 0.8054601550102234)]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv_from_text.most_similar_cosmul(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = wv_from_text.get_vector('Moreau'.lower() if 'Moreau'.lower() in wv_from_text.key_to_index else find_closest_word('Moreau'.lower(), word_list) , norm=True) \n",
    "b = wv_from_text.get_vector('Seminary'.lower() if 'Seminary'.lower() in wv_from_text.key_to_index else find_closest_word('Seminary'.lower(), word_list), norm=True)\n",
    "c = (a+b)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preliminary\n",
      "premilinary: 0.7334\n"
     ]
    }
   ],
   "source": [
    "closest_word = find_closest_word(\"Moreau Seminary\", word_list)\n",
    "print(closest_word)\n",
    "result = wv_from_text.similar_by_word(closest_word)\n",
    "most_similar_key, similarity = result[0]\n",
    "print(f\"{most_similar_key}: {similarity:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: The Apollo program, also known as Project Apollo, was the third United States human spaceflight program carried out by the National Aeronautics and Space Administration (NASA), which accomplished landing the first humans on the Moon from 1969 to 1972. First conceived during Dwight D. Eisenhower's administration as a three-man spacecraft to follow the one-man Project Mercury which put the first Americans in space, Apollo was later dedicated to President John F. Kennedy's national goal of \"landing a man on the Moon and returning him safely to the Earth\" by the end of the 1960s, which he proposed in a May 25, 1961, address to Congress. Project Mercury was followed by the two-man Project Gemini (1962–66). The first manned flight of Apollo was in 1968.\n",
      "Question: What program was created to carry out these projects and missions?\n",
      "Answer: National Aeronautics and Space Administration (NASA)\n",
      "--------------------------------------------------\n",
      "Context: The first four Saturn I test flights were launched from LC-34, with only live first stages, carrying dummy upper stages filled with water. The first flight with a live S-IV was launched from LC-37. This was followed by five launches of boilerplate CSMs (designated AS-101 through AS-105) into orbit in 1964 and 1965. The last three of these further supported the Apollo program by also carrying Pegasus satellites, which verified the safety of the translunar environment by measuring the frequency and severity of micrometeorite impacts.\n",
      "Question: What did Pegasus satellites measure to propose the safety of the environment in space and on the moon?\n",
      "Answer: frequency and severity of micrometeorite impacts\n",
      "--------------------------------------------------\n",
      "Context: Two Block I CSMs were launched from LC-34 on suborbital flights in 1966 with the Saturn IB. The first, AS-201 launched on February 26, reached an altitude of 265.7 nautical miles (492.1 km) and splashed down 4,577 nautical miles (8,477 km) downrange in the Atlantic ocean. The second, AS-202 on August 25, reached 617.1 nautical miles (1,142.9 km) altitude and was recovered 13,900 nautical miles (25,700 km) downrange in the Pacific ocean. These flights validated the Service Module engine and the Command Module heat shield.\n",
      "Question: How far did the second block, AS-202, travel downrange in km? \n",
      "Answer: 25,700 km\n",
      "--------------------------------------------------\n",
      "Context: Almost all the rocks show evidence of impact process effects. Many samples appear to be pitted with micrometeoroid impact craters, which is never seen on Earth rocks, due to the thick atmosphere. Many show signs of being subjected to high pressure shock waves that are generated during impact events. Some of the returned samples are of impact melt (materials melted near an impact crater.) All samples returned from the Moon are highly brecciated as a result of being subjected to multiple impact events.\n",
      "Question: What do a great majority of rocks sampled from the moon show?\n",
      "Answer: impact process effects\n",
      "--------------------------------------------------\n",
      "Context: One of its earliest massive implementations was brought about by Egyptians against the British occupation in the 1919 Revolution. Civil disobedience is one of the many ways people have rebelled against what they deem to be unfair laws. It has been used in many nonviolent resistance movements in India (Gandhi's campaigns for independence from the British Empire), in Czechoslovakia's Velvet Revolution and in East Germany to oust their communist governments, In South Africa in the fight against apartheid, in the American Civil Rights Movement, in the Singing Revolution to bring independence to the Baltic countries from the Soviet Union, recently with the 2003 Rose Revolution in Georgia and the 2004 Orange Revolution in Ukraine, among other various movements worldwide.\n",
      "Question: Who was one of the earliest examples of Civil Disobedience against?\n",
      "Answer: the British\n",
      "--------------------------------------------------\n",
      "Context: Civil disobedience is usually defined as pertaining to a citizen's relation to the state and its laws, as distinguished from a constitutional impasse in which two public agencies, especially two equally sovereign branches of government, conflict. For instance, if the head of government of a country were to refuse to enforce a decision of that country's highest court, it would not be civil disobedience, since the head of government would be acting in her or his capacity as public official rather than private citizen.\n",
      "Question: How is civil disobedience typically defined in connection of the citizen's?\n",
      "Answer: to the state and its laws\n",
      "--------------------------------------------------\n",
      "Context: It is usually recognized that lawbreaking, if it is not done publicly, at least must be publicly announced in order to constitute civil disobedience. But Stephen Eilmann argues that if it is necessary to disobey rules that conflict with morality, we might ask why disobedience should take the form of public civil disobedience rather than simply covert lawbreaking. If a lawyer wishes to help a client overcome legal obstacles to securing her or his natural rights, he might, for instance, find that assisting in fabricating evidence or committing perjury is more effective than open disobedience. This assumes that common morality does not have a prohibition on deceit in such situations. The Fully Informed Jury Association's publication \"A Primer for Prospective Jurors\" notes, \"Think of the dilemma faced by German citizens when Hitler's secret police demanded to know if they were hiding a Jew in their house.\" By this definition, civil disobedience could be traced back to the Book of Exodus, where Shiphrah and Puah refused a direct order of Pharaoh but misrepresented how they did it. (Exodus 1: 15-19)\n",
      "Question: Stephen Eildmann cites the oldest known example of civil disobedience in what part of the bible? \n",
      "Answer: (Exodus 1: 15-19)\n",
      "--------------------------------------------------\n",
      "Context: There have been debates as to whether civil disobedience must necessarily be non-violent. Black's Law Dictionary includes non-violence in its definition of civil disobedience. Christian Bay's encyclopedia article states that civil disobedience requires \"carefully chosen and legitimate means,\" but holds that they do not have to be non-violent. It has been argued that, while both civil disobedience and civil rebellion are justified by appeal to constitutional defects, rebellion is much more destructive; therefore, the defects justifying rebellion must be much more serious than those justifying disobedience, and if one cannot justify civil rebellion, then one cannot justify a civil disobedients' use of force and violence and refusal to submit to arrest. Civil disobedients' refraining from violence is also said to help preserve society's tolerance of civil disobedience.\n",
      "Question: Why is it preferred that civil disobedience is non violent?\n",
      "Answer: help preserve society's tolerance of civil disobedience\n",
      "--------------------------------------------------\n",
      "Context: Non-revolutionary civil disobedience is a simple disobedience of laws on the grounds that they are judged \"wrong\" by an individual conscience, or as part of an effort to render certain laws ineffective, to cause their repeal, or to exert pressure to get one's political wishes on some other issue. Revolutionary civil disobedience is more of an active attempt to overthrow a government (or to change cultural traditions, social customs, religious beliefs, etc...revolution doesn't have to be political, i.e. \"cultural revolution\", it simply implies sweeping and widespread change to a section of the social fabric). Gandhi's acts have been described as revolutionary civil disobedience. It has been claimed that the Hungarians under Ferenc Deák directed revolutionary civil disobedience against the Austrian government. Thoreau also wrote of civil disobedience accomplishing \"peaceable revolution.\" Howard Zinn, Harvey Wheeler, and others have identified the right espoused in The Declaration of Independence to \"alter or abolish\" an unjust government to be a principle of civil disobedience. \n",
      "Question: What is the goal of individual civil disobedience?\n",
      "Answer: render certain laws ineffective\n",
      "--------------------------------------------------\n",
      "Context: The earliest recorded incidents of collective civil disobedience took place during the Roman Empire[citation needed]. Unarmed Jews gathered in the streets to prevent the installation of pagan images in the Temple in Jerusalem.[citation needed][original research?] In modern times, some activists who commit civil disobedience as a group collectively refuse to sign bail until certain demands are met, such as favorable bail conditions, or the release of all the activists. This is a form of jail solidarity.[page needed] There have also been many instances of solitary civil disobedience, such as that committed by Thoreau, but these sometimes go unnoticed. Thoreau, at the time of his arrest, was not yet a well-known author, and his arrest was not covered in any newspapers in the days, weeks and months after it happened. The tax collector who arrested him rose to higher political office, and Thoreau's essay was not published until after the end of the Mexican War.\n",
      "Question: How long did it take for Thoreau's disobedience to be known?\n",
      "Answer: until after the end of the Mexican War\n",
      "--------------------------------------------------\n",
      "Context: Civil disobedients have chosen a variety of different illegal acts. Bedau writes, \"There is a whole class of acts, undertaken in the name of civil disobedience, which, even if they were widely practiced, would in themselves constitute hardly more than a nuisance (e.g. trespassing at a nuclear-missile installation)...Such acts are often just a harassment and, at least to the bystander, somewhat inane...The remoteness of the connection between the disobedient act and the objectionable law lays such acts open to the charge of ineffectiveness and absurdity.\" Bedau also notes, though, that the very harmlessness of such entirely symbolic illegal protests toward public policy goals may serve a propaganda purpose. Some civil disobedients, such as the proprietors of illegal medical cannabis dispensaries and Voice in the Wilderness, which brought medicine to Iraq without the permission of the U.S. Government, directly achieve a desired social goal (such as the provision of medication to the sick) while openly breaking the law. Julia Butterfly Hill lived in Luna, a 180-foot (55 m)-tall, 600-year-old California Redwood tree for 738 days, successfully preventing it from being cut down.\n",
      "Question: Such protests are usually considered to be what type?\n",
      "Answer: entirely symbolic\n",
      "--------------------------------------------------\n",
      "Context: Some forms of civil disobedience, such as illegal boycotts, refusals to pay taxes, draft dodging, distributed denial-of-service attacks, and sit-ins, make it more difficult for a system to function. In this way, they might be considered coercive. Brownlee notes that \"although civil disobedients are constrained in their use of coercion by their conscientious aim to engage in moral dialogue, nevertheless they may find it necessary to employ limited coercion in order to get their issue onto the table.\" The Plowshares organization temporarily closed GCSB Waihopai by padlocking the gates and using sickles to deflate one of the large domes covering two satellite dishes.\n",
      "Question: How can you protest against big companies in a non violent way?\n",
      "Answer: padlocking the gates\n",
      "--------------------------------------------------\n",
      "Context: Many of the same decisions and principles that apply in other criminal investigations and arrests arise also in civil disobedience cases. For example, the suspect may need to decide whether or not to grant a consent search of his property, and whether or not to talk to police officers. It is generally agreed within the legal community, and is often believed within the activist community, that a suspect's talking to criminal investigators can serve no useful purpose, and may be harmful. However, some civil disobedients have nonetheless found it hard to resist responding to investigators' questions, sometimes due to a lack of understanding of the legal ramifications, or due to a fear of seeming rude. Also, some civil disobedients seek to use the arrest as an opportunity to make an impression on the officers. Thoreau wrote, \"My civil neighbor, the tax-gatherer, is the very man I have to deal with--for it is, after all, with men and not with parchment that I quarrel--and he has voluntarily chosen to be an agent of the government. How shall he ever know well that he is and does as an officer of the government, or as a man, until he is obliged to consider whether he will treat me, his neighbor, for whom he has respect, as a neighbor and well-disposed man, or as a maniac and disturber of the peace, and see if he can get over this obstruction to his neighborliness without a ruder and more impetuous thought or speech corresponding with his action.\"\n",
      "Question: What is a way you can show police officers civil disobedience ?\n",
      "Answer: not to grant a consent search\n",
      "--------------------------------------------------\n",
      "Context: Some civil disobedients feel it is incumbent upon them to accept punishment because of their belief in the validity of the social contract, which is held to bind all to obey the laws that a government meeting certain standards of legitimacy has established, or else suffer the penalties set out in the law. Other civil disobedients who favor the existence of government still don't believe in the legitimacy of their particular government, or don't believe in the legitimacy of a particular law it has enacted. And still other civil disobedients, being anarchists, don't believe in the legitimacy of any government, and therefore see no need to accept punishment for a violation of criminal law that does not infringe the rights of others.\n",
      "Question: Anarchists do not want to accept punishment for what reason?\n",
      "Answer: does not infringe the rights of others\n",
      "--------------------------------------------------\n",
      "Context: When the Committee for Non-Violent Action sponsored a protest in August 1957, at the Camp Mercury nuclear test site near Las Vegas, Nevada, 13 of the protesters attempted to enter the test site knowing that they faced arrest. At a pre-arranged announced time, one at a time they stepped across the \"line\" and were immediately arrested. They were put on a bus and taken to the Nye County seat of Tonopah, Nevada, and arraigned for trial before the local Justice of the Peace, that afternoon. A well known civil rights attorney, Francis Heisler, had volunteered to defend the arrested persons, advising them to plead \"nolo contendere\", as an alternative to pleading either guilty or not-guilty. The arrested persons were found \"guilty,\" nevertheless, and given suspended sentences, conditional on their not reentering the test site grounds.[citation needed]\n",
      "Question: What was the civil disobedience performed at the test site?\n",
      "Answer: tempted to enter the test site\n",
      "--------------------------------------------------\n",
      "Context: When the Committee for Non-Violent Action sponsored a protest in August 1957, at the Camp Mercury nuclear test site near Las Vegas, Nevada, 13 of the protesters attempted to enter the test site knowing that they faced arrest. At a pre-arranged announced time, one at a time they stepped across the \"line\" and were immediately arrested. They were put on a bus and taken to the Nye County seat of Tonopah, Nevada, and arraigned for trial before the local Justice of the Peace, that afternoon. A well known civil rights attorney, Francis Heisler, had volunteered to defend the arrested persons, advising them to plead \"nolo contendere\", as an alternative to pleading either guilty or not-guilty. The arrested persons were found \"guilty,\" nevertheless, and given suspended sentences, conditional on their not reentering the test site grounds.[citation needed]\n",
      "Question: What was the result of the disobedience protesting the nuclear site?\n",
      "Answer: arrested\n",
      "--------------------------------------------------\n",
      "Context: Steven Barkan writes that if defendants plead not guilty, \"they must decide whether their primary goal will be to win an acquittal and avoid imprisonment or a fine, or to use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case and their reasons for breaking the law via civil disobedience.\" A technical defense may enhance the chances for acquittal but make for more boring proceedings and reduced press coverage. During the Vietnam War era, the Chicago Eight used a political defense, while Benjamin Spock used a technical defense. In countries such as the United States whose laws guarantee the right to a jury trial but do not excuse lawbreaking for political purposes, some civil disobedients seek jury nullification. Over the years, this has been made more difficult by court decisions such as Sparf v. United States, which held that the judge need not inform jurors of their nullification prerogative, and United States v. Dougherty, which held that the judge need not allow defendants to openly seek jury nullification.\n",
      "Question: What is the primary goal of pleading not guilty when arrested for Civil Disobedience?\n",
      "Answer: acquittal and avoid imprisonment\n",
      "--------------------------------------------------\n",
      "Context: One theory is that, while disobedience may be helpful, any great amount of it would undermine the law by encouraging general disobedience which is neither conscientious nor of social benefit. Therefore, conscientious lawbreakers must be punished. Michael Bayles argues that if a person violates a law in order to create a test case as to the constitutionality of a law, and then wins his case, then that act did not constitute civil disobedience. It has also been argued that breaking the law for self-gratification, as in the case of a homosexual or cannabis user who does not direct his act at securing the repeal of amendment of the law, is not civil disobedience. Likewise, a protestor who attempts to escape punishment by committing the crime covertly and avoiding attribution, or by denying having committed the crime, or by fleeing the jurisdiction, is generally viewed as not being a civil disobedient.\n",
      "Question: Why should someone not commit a crime when they are protesting?\n",
      "Answer: not being a civil disobedient\n",
      "--------------------------------------------------\n",
      "Context: The variant forms of the name of the Rhine in modern languages are all derived from the Gaulish name Rēnos, which was adapted in Roman-era geography (1st century BC) as Greek Ῥῆνος (Rhēnos), Latin Rhenus.[note 3] The spelling with Rh- in English Rhine as well as in German Rhein and French Rhin is due to the influence of Greek orthography, while the vocalisation -i- is due to the Proto-Germanic adoption of the Gaulish name as *Rīnaz, via Old Frankish giving Old English Rín, Old High German Rīn, Dutch Rijn (formerly also spelled Rhijn)). The diphthong in modern German Rhein (also adopted in Romansh Rein, Rain) is a Central German development of the early modern period, the Alemannic name Rī(n) retaining the older vocalism,[note 4] as does Ripuarian Rhing, while Palatine has diphthongized Rhei, Rhoi. Spanish is with French in adopting the Germanic vocalism Rin-, while Italian, Occitan and Portuguese retain the Latin Ren-.\n",
      "Question: What does the name The Rhine come from? \n",
      "Answer: Gaulish name Rēnos\n",
      "--------------------------------------------------\n",
      "Context: Imperialism is defined as \"A policy of extending a country’s power and influence through diplomacy or military force.\" Imperialism is particularly focused on the control that one group, often a state power, has on another group of people. This is often through various forms of \"othering\" (see other) based on racial, religious, or cultural stereotypes. There are \"formal\" or \"informal\" imperialisms. \"Formal imperialism\" is defined as \"physical control or full-fledged colonial rule\". \"Informal imperialism\" is less direct; however, it is still a powerful form of dominance.\n",
      "Question: colonial rule would be considered what type of imperialism?\n",
      "Answer: \"Formal imperialism\"\n",
      "--------------------------------------------------\n",
      "Context: Imperialism and colonialism both dictate the political and economic advantage over a land and the indigenous populations they control, yet scholars sometimes find it difficult to illustrate the difference between the two. Although imperialism and colonialism focus on the suppression of an other, if colonialism refers to the process of a country taking physical control of another, imperialism refers to the political and monetary dominance, either formally or informally. Colonialism is seen to be the architect deciding how to start dominating areas and then imperialism can be seen as creating the idea behind conquest cooperating with colonialism. Colonialism is when the imperial nation begins a conquest over an area and then eventually is able to rule over the areas the previous nation had controlled. Colonialism's core meaning is the exploitation of the valuable assets and supplies of the nation that was conquered and the conquering nation then gaining the benefits from the spoils of the war. The meaning of imperialism is to create an empire, by conquering the other state's lands and therefore increasing its own dominance. Colonialism is the builder and preserver of the colonial possessions in an area by a population coming from a foreign region. Colonialism can completely change the existing social structure, physical structure and economics of an area; it is not unusual that the characteristics of the conquering peoples are inherited by the conquered indigenous populations.\n",
      "Question: Colonialism often means a country doing what?\n",
      "Answer: taking physical control of another\n",
      "--------------------------------------------------\n",
      "Context: Imperialism and colonialism both dictate the political and economic advantage over a land and the indigenous populations they control, yet scholars sometimes find it difficult to illustrate the difference between the two. Although imperialism and colonialism focus on the suppression of an other, if colonialism refers to the process of a country taking physical control of another, imperialism refers to the political and monetary dominance, either formally or informally. Colonialism is seen to be the architect deciding how to start dominating areas and then imperialism can be seen as creating the idea behind conquest cooperating with colonialism. Colonialism is when the imperial nation begins a conquest over an area and then eventually is able to rule over the areas the previous nation had controlled. Colonialism's core meaning is the exploitation of the valuable assets and supplies of the nation that was conquered and the conquering nation then gaining the benefits from the spoils of the war. The meaning of imperialism is to create an empire, by conquering the other state's lands and therefore increasing its own dominance. Colonialism is the builder and preserver of the colonial possessions in an area by a population coming from a foreign region. Colonialism can completely change the existing social structure, physical structure and economics of an area; it is not unusual that the characteristics of the conquering peoples are inherited by the conquered indigenous populations.\n",
      "Question: What is colonialism's core meaning?\n",
      "Answer: exploitation\n",
      "--------------------------------------------------\n",
      "Context: The war was fought primarily along the frontiers between New France and the British colonies, from Virginia in the South to Nova Scotia in the North. It began with a dispute over control of the confluence of the Allegheny and Monongahela rivers, called the Forks of the Ohio, and the site of the French Fort Duquesne and present-day Pittsburgh, Pennsylvania. The dispute erupted into violence in the Battle of Jumonville Glen in May 1754, during which Virginia militiamen under the command of 22-year-old George Washington ambushed a French patrol.\n",
      "Question: Where was war fought?\n",
      "Answer: primarily along the frontiers between New France and the British colonies\n",
      "--------------------------------------------------\n",
      "Context: In 1755, six colonial governors in North America met with General Edward Braddock, the newly arrived British Army commander, and planned a four-way attack on the French. None succeeded and the main effort by Braddock was a disaster; he was defeated in the Battle of the Monongahela on July 9, 1755 and died a few days later. British operations in 1755, 1756 and 1757 in the frontier areas of Pennsylvania and New York all failed, due to a combination of poor management, internal divisions, and effective Canadian scouts, French regular forces, and Indian warrior allies. In 1755, the British captured Fort Beauséjour on the border separating Nova Scotia from Acadia; soon afterward they ordered the expulsion of the Acadians. Orders for the deportation were given by William Shirley, Commander-in-Chief, North America, without direction from Great Britain. The Acadians, both those captured in arms and those who had sworn the loyalty oath to His Britannic Majesty, were expelled. Native Americans were likewise driven off their land to make way for settlers from New England.\n",
      "Question: How successful was initial effort by Braddock?\n",
      "Answer: disaster; he was defeated in the Battle of the Monongahela\n",
      "--------------------------------------------------\n",
      "Context: In 1755, six colonial governors in North America met with General Edward Braddock, the newly arrived British Army commander, and planned a four-way attack on the French. None succeeded and the main effort by Braddock was a disaster; he was defeated in the Battle of the Monongahela on July 9, 1755 and died a few days later. British operations in 1755, 1756 and 1757 in the frontier areas of Pennsylvania and New York all failed, due to a combination of poor management, internal divisions, and effective Canadian scouts, French regular forces, and Indian warrior allies. In 1755, the British captured Fort Beauséjour on the border separating Nova Scotia from Acadia; soon afterward they ordered the expulsion of the Acadians. Orders for the deportation were given by William Shirley, Commander-in-Chief, North America, without direction from Great Britain. The Acadians, both those captured in arms and those who had sworn the loyalty oath to His Britannic Majesty, were expelled. Native Americans were likewise driven off their land to make way for settlers from New England.\n",
      "Question: Why did British operation fail in 1755, 56, 57?\n",
      "Answer: combination of poor management, internal divisions, and effective Canadian scouts, French regular forces, and Indian warrior allies\n",
      "--------------------------------------------------\n",
      "Context: The conflict is known by multiple names. In British America, wars were often named after the sitting British monarch, such as King William's War or Queen Anne's War. As there had already been a King George's War in the 1740s, British colonists named the second war in King George's reign after their opponents, and it became known as the French and Indian War. This traditional name continues as the standard in the United States, but it obscures the fact that Indians fought on both sides of the conflict, and that this was part of the Seven Years' War, a much larger conflict between France and Great Britain. American historians generally use the traditional name or sometimes the Seven Years' War. Other, less frequently used names for the war include the Fourth Intercolonial War and the Great War for the Empire.\n",
      "Question: What is the confusion of the French and Indian war?\n",
      "Answer: Indians fought on both sides of the conflict, and that this was part of the Seven Years' War\n",
      "--------------------------------------------------\n",
      "Context: The French population numbered about 75,000 and was heavily concentrated along the St. Lawrence River valley, with some also in Acadia (present-day New Brunswick and parts of Nova Scotia, including Île Royale (present-day Cape Breton Island)). Fewer lived in New Orleans, Biloxi, Mississippi, Mobile, Alabama and small settlements in the Illinois Country, hugging the east side of the Mississippi River and its tributaries. French fur traders and trappers traveled throughout the St. Lawrence and Mississippi watersheds, did business with local tribes, and often married Indian women. Traders married daughters of chiefs, creating high-ranking unions.\n",
      "Question: Where did French fur trappers travel?\n",
      "Answer: St. Lawrence and Mississippi watersheds, did business with local tribes, and often married Indian women\n",
      "--------------------------------------------------\n",
      "Context: Céloron's expedition force consisted of about 200 Troupes de la marine and 30 Indians. The expedition covered about 3,000 miles (4,800 km) between June and November 1749. It went up the St. Lawrence, continued along the northern shore of Lake Ontario, crossed the portage at Niagara, and followed the southern shore of Lake Erie. At the Chautauqua Portage (near present-day Barcelona, New York), the expedition moved inland to the Allegheny River, which it followed to the site of present-day Pittsburgh. There Céloron buried lead plates engraved with the French claim to the Ohio Country. Whenever he encountered British merchants or fur-traders, Céloron informed them of the French claims on the territory and told them to leave.\n",
      "Question: How did Celeron handle business on trip?\n",
      "Answer: British merchants or fur-traders, Céloron informed them of the French claims on the territory and told them to leave.\n",
      "--------------------------------------------------\n",
      "Context: When Céloron's expedition arrived at Logstown, the Native Americans in the area informed Céloron that they owned the Ohio Country and that they would trade with the British regardless of the French. Céloron continued south until his expedition reached the confluence of the Ohio and the Miami rivers, which lay just south of the village of Pickawillany, the home of the Miami chief known as \"Old Briton\". Céloron threatened \"Old Briton\" with severe consequences if he continued to trade with the British. \"Old Briton\" ignored the warning. Disappointed, Céloron returned to Montreal in November 1749.\n",
      "Question: How did Celeron handle meeting with Old Briton?\n",
      "Answer: threatened \"Old Briton\" with severe consequences if he continued to trade with the British\n",
      "--------------------------------------------------\n",
      "Context: On March 17, 1752, the Governor-General of New France, Marquis de la Jonquière, died and was temporarily replaced by Charles le Moyne de Longueuil. His permanent replacement, the Marquis Duquesne, did not arrive in New France until 1752 to take over the post. The continuing British activity in the Ohio territories prompted Longueuil to dispatch another expedition to the area under the command of Charles Michel de Langlade, an officer in the Troupes de la Marine. Langlade was given 300 men, including French-Canadians and warriors of the Ottawa. His objective was to punish the Miami people of Pickawillany for not following Céloron's orders to cease trading with the British. On June 21, the French war party attacked the trading centre at Pickawillany, capturing three traders and killing 14 people of the Miami nation, including Old Briton. He was reportedly ritually cannibalized by some aboriginal members of the expedition.\n",
      "Question: Was was the plan for Langlades mission?\n",
      "Answer: punish the Miami people of Pickawillany for not following Céloron's orders to cease trading with the British\n",
      "--------------------------------------------------\n",
      "Context: The Iroquois sent runners to the manor of William Johnson in upstate New York. The British Superintendent for Indian Affairs in the New York region and beyond, Johnson was known to the Iroquois as Warraghiggey, meaning \"He who does great things.\" He spoke their languages and had become a respected honorary member of the Iroquois Confederacy in the area. In 1746, Johnson was made a colonel of the Iroquois. Later he was commissioned as a colonel of the Western New York Militia. They met at Albany, New York with Governor Clinton and officials from some of the other American colonies. Mohawk Chief Hendrick, Speaker of their tribal council, insisted that the British abide by their obligations and block French expansion. When Clinton did not respond to his satisfaction, Chief Hendrick said that the \"Covenant Chain\", a long-standing friendly relationship between the Iroquois Confederacy and the British Crown, was broken.\n",
      "Question: What title did Iroquois give Johnson?\n",
      "Answer: colonel of the Iroquois\n",
      "--------------------------------------------------\n",
      "Context: Jacques Legardeur de Saint-Pierre, who succeeded Marin as commander of the French forces after the latter died on October 29, invited Washington to dine with him. Over dinner, Washington presented Saint-Pierre with the letter from Dinwiddie demanding an immediate French withdrawal from the Ohio Country. Saint-Pierre said, \"As to the Summons you send me to retire, I do not think myself obliged to obey it.\" He told Washington that France's claim to the region was superior to that of the British, since René-Robert Cavelier, Sieur de La Salle had explored the Ohio Country nearly a century earlier.\n",
      "Question: What letter did Washington present to  Saint-Pierre ?\n",
      "Answer: Dinwiddie demanding an immediate French withdrawal from the Ohio Country\n",
      "--------------------------------------------------\n",
      "Context: Jacques Legardeur de Saint-Pierre, who succeeded Marin as commander of the French forces after the latter died on October 29, invited Washington to dine with him. Over dinner, Washington presented Saint-Pierre with the letter from Dinwiddie demanding an immediate French withdrawal from the Ohio Country. Saint-Pierre said, \"As to the Summons you send me to retire, I do not think myself obliged to obey it.\" He told Washington that France's claim to the region was superior to that of the British, since René-Robert Cavelier, Sieur de La Salle had explored the Ohio Country nearly a century earlier.\n",
      "Question: Why did French feel they had right to Ohio claim?\n",
      "Answer: France's claim to the region was superior to that of the British\n",
      "--------------------------------------------------\n",
      "Context: After Washington had returned to Williamsburg, Dinwiddie ordered him to lead a larger force to assist Trent in his work. While en route, Washington learned of Trent's retreat. Since Tanaghrisson had promised support to the British, Washington continued toward Fort Duquesne and met with the Mingo leader. Learning of a French scouting party in the area, Washington, with Tanaghrisson and his party, surprised the Canadians on May 28 in what became known as the Battle of Jumonville Glen. They killed many of the Canadians, including their commanding officer, Joseph Coulon de Jumonville, whose head was reportedly split open by Tanaghrisson with a tomahawk. The historian Fred Anderson suggests that Tanaghrisson was acting to gain the support of the British and regain authority over his own people. They had been inclined to support the French, with whom they had long trading relationships. One of Tanaghrisson's men told Contrecoeur that Jumonville had been killed by British musket fire.\n",
      "Question: Why was Tanaghrisson supporting British efforts?\n",
      "Answer: regain authority over his own people. They had been inclined to support the French, with whom they had long trading relationships\n",
      "--------------------------------------------------\n",
      "Context: Following the death of Braddock, William Shirley assumed command of British forces in North America. At a meeting in Albany in December 1755, he laid out his plans for 1756. In addition to renewing the efforts to capture Niagara, Crown Point and Duquesne, he proposed attacks on Fort Frontenac on the north shore of Lake Ontario and an expedition through the wilderness of the Maine district and down the Chaudière River to attack the city of Quebec. Bogged down by disagreements and disputes with others, including William Johnson and New York's Governor Sir Charles Hardy, Shirley's plan had little support.\n",
      "Question: Where was Shirley planning an expedition?\n",
      "Answer: through the wilderness of the Maine district and down the Chaudière River to attack the city of Quebec\n",
      "--------------------------------------------------\n",
      "Context: The third invasion was stopped with the improbable French victory in the Battle of Carillon, in which 3,600 Frenchmen famously and decisively defeated Abercrombie's force of 18,000 regulars, militia and Native American allies outside the fort the French called Carillon and the British called Ticonderoga. Abercrombie saved something from the disaster when he sent John Bradstreet on an expedition that successfully destroyed Fort Frontenac, including caches of supplies destined for New France's western forts and furs destined for Europe. Abercrombie was recalled and replaced by Jeffery Amherst, victor at Louisbourg.\n",
      "Question: What impact did this loss have on Abercrombie?\n",
      "Answer: recalled and replaced by Jeffery Amherst, victor at Louisbourg.\n",
      "--------------------------------------------------\n",
      "Context: For many native populations, the elimination of French power in North America meant the disappearance of a strong ally and counterweight to British expansion, leading to their ultimate dispossession. The Ohio Country was particularly vulnerable to legal and illegal settlement due to the construction of military roads to the area by Braddock and Forbes. Although the Spanish takeover of the Louisiana territory (which was not completed until 1769) had modest repercussions, the British takeover of Spanish Florida resulted in the westward migration of tribes that did not want to do business with the British, and a rise in tensions between the Choctaw and the Creek, historic enemies whose divisions the British at times exploited. The change of control in Florida also prompted most of its Spanish Catholic population to leave. Most went to Cuba, including the entire governmental records from St. Augustine, although some Christianized Yamasee were resettled to the coast of Mexico.\n",
      "Question: What made Ohio Country vulnerable?\n",
      "Answer: military roads to the area by Braddock and Forbes\n",
      "--------------------------------------------------\n",
      "Context: Newton's First Law of Motion states that objects continue to move in a state of constant velocity unless acted upon by an external net force or resultant force. This law is an extension of Galileo's insight that constant velocity was associated with a lack of net force (see a more detailed description of this below). Newton proposed that every object with mass has an innate inertia that functions as the fundamental equilibrium \"natural state\" in place of the Aristotelian idea of the \"natural state of rest\". That is, the first law contradicts the intuitive Aristotelian belief that a net force is required to keep an object moving with constant velocity. By making rest physically indistinguishable from non-zero constant velocity, Newton's First Law directly connects inertia with the concept of relative velocities. Specifically, in systems where objects are moving with different velocities, it is impossible to determine which object is \"in motion\" and which object is \"at rest\". In other words, to phrase matters more technically, the laws of physics are the same in every inertial frame of reference, that is, in all frames related by a Galilean transformation.\n",
      "Question: What are the laws of physics of Galileo, in reference to objest in motion and rest?\n",
      "Answer: the same\n",
      "--------------------------------------------------\n",
      "Context: This means that in a closed system of particles, there are no internal forces that are unbalanced. That is, the action-reaction force shared between any two objects in a closed system will not cause the center of mass of the system to accelerate. The constituent objects only accelerate with respect to each other, the system itself remains unaccelerated. Alternatively, if an external force acts on the system, then the center of mass will experience an acceleration proportional to the magnitude of the external force divided by the mass of the system.:19-1\n",
      "Question: In what kind of system of particles are there no unbalanced iinternal forces?\n",
      "Answer: closed\n",
      "--------------------------------------------------\n",
      "Context: However, already in quantum mechanics there is one \"caveat\", namely the particles acting onto each other do not only possess the spatial variable, but also a discrete intrinsic angular momentum-like variable called the \"spin\", and there is the Pauli principle relating the space and the spin variables. Depending on the value of the spin, identical particles split into two different classes, fermions and bosons. If two identical fermions (e.g. electrons) have a symmetric spin function (e.g. parallel spins) the spatial variables must be antisymmetric (i.e. they exclude each other from their places much as if there was a repulsive force), and vice versa, i.e. for antiparallel spins the position variables must be symmetric (i.e. the apparent force must be attractive). Thus in the case of two fermions there is a strictly negative correlation between spatial and spin variables, whereas for two bosons (e.g. quanta of electromagnetic waves, photons) the correlation is strictly positive.\n",
      "Question: What value does the seperating into fermions and bosons depend?\n",
      "Answer: spin\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "for i, row in squad[squad.answer_cnt == 5].iterrows():\n",
    "    print(\"Context: \" + row[\"context\"])\n",
    "    print(\"Question: \" + row[\"question\"])\n",
    "    print(\"Answer: \" + row[\"answers\"][\"text\"][0])\n",
    "    print(\"-\"*50)\n",
    "    if i == 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_short_context(text: str, answer: ) -> str:\n",
    "    sentences = sent_tokenize(text)\n",
    "    if len(sentences) > 1:\n",
    "        return \" \".join(sentences[:2])\n",
    "    else:\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_sents = sent_tokenize(original_context)\n",
    "    for idx, sent in enumerate(original_sents):\n",
    "        if original_answer in sent:\n",
    "            answer_sent_idx = idx\n",
    "start_idx, end_idx = max(0, answer_sent_idx-3), min(len(original_sents), answer_sent_idx+4)\n",
    "short_context = \" \".join(original_sents[start_idx:end_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Glove Model Loaded\n"
     ]
    }
   ],
   "source": [
    "import gensim.downloader as api\n",
    "model = api.load(\"glove-wiki-gigaword-300\")\n",
    "print(\"Glove Model Loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from exp_adv_case import TASA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(load_dataset(\"Seongill/squad_conflict_v2_under_150_with_substitution_chunked\", split=\"train\").select(range(10)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "find_similar_entity() missing 1 required positional argument: 'wv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[32], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mTASA\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/rag/exp_adv_case.py:155\u001b[0m, in \u001b[0;36mTASA\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m    153\u001b[0m new_answer \u001b[38;5;241m=\u001b[39m row[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msimilar_answer\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    154\u001b[0m question_doc, answer_sentence_doc \u001b[38;5;241m=\u001b[39m question_output[question], answer_output[answer_sentence]\n\u001b[0;32m--> 155\u001b[0m new_answer_sentence \u001b[38;5;241m=\u001b[39m \u001b[43mmake_adversarial_sentence\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_answer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43manswer_sentence\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquestion_doc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43manswer_sentence_doc\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    156\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mQuestion: \u001b[39m\u001b[38;5;124m\"\u001b[39m, question)\n\u001b[1;32m    157\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAnswer: \u001b[39m\u001b[38;5;124m\"\u001b[39m, answer)\n",
      "File \u001b[0;32m~/rag/exp_adv_case.py:112\u001b[0m, in \u001b[0;36mmake_adversarial_sentence\u001b[0;34m(new_answer, answer, answer_sentence, question_doc, answer_sentence_doc)\u001b[0m\n\u001b[1;32m    110\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m ent \u001b[38;5;129;01min\u001b[39;00m answer_sentence_doc\u001b[38;5;241m.\u001b[39ments:\n\u001b[1;32m    111\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m ent \u001b[38;5;129;01mand\u001b[39;00m (ent\u001b[38;5;241m.\u001b[39mtext \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m keywords) \u001b[38;5;129;01mand\u001b[39;00m (ent\u001b[38;5;241m.\u001b[39mtext \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m replaced_entity):\n\u001b[0;32m--> 112\u001b[0m         replaced_entity[ent\u001b[38;5;241m.\u001b[39mtext] \u001b[38;5;241m=\u001b[39m \u001b[43mfind_similar_entity\u001b[49m\u001b[43m(\u001b[49m\u001b[43ment\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlabel_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43ment\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    113\u001b[0m replaced_words \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m()\n\u001b[1;32m    114\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m answer_sentence_doc:\n",
      "\u001b[0;31mTypeError\u001b[0m: find_similar_entity() missing 1 required positional argument: 'wv'"
     ]
    }
   ],
   "source": [
    "TASA(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "exp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}