1-0_tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "handled-marine",
   "metadata": {},
   "source": [
    "# tokenization\n",
    "\n",
    "토큰화 되는 squad_convert_examples_to_features 함수에 대한 설명 및 여기서 어떻게 토큰화 되는지에 대한 설명\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "proof-arctic",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers.data.processors.squad import _is_whitespace, whitespace_tokenize, _improve_answer_span, _new_check_is_max_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "later-adjustment",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "1. 먼저 json 파일에서 파싱된 정보를 저장하고\n",
    "2. 정답에 해당하는 answer text 를 가지고 end position 을 찾아 저장해둡니다\n",
    "3. 공백 기준으로 토큰화를 진행해둡니다\n",
    "\"\"\" \n",
    "\n",
    "class SquadExample:\n",
    "    \"\"\"\n",
    "    A single training/test example for the Squad dataset, as loaded from disk.\n",
    "\n",
    "    Args:\n",
    "        qas_id: The example's unique identifier\n",
    "        question_text: The question string\n",
    "        context_text: The context string\n",
    "        answer_text: The answer string\n",
    "        start_position_character: The character position of the start of the answer\n",
    "        title: The title of the example\n",
    "        answers: None by default, this is used during evaluation. Holds answers as well as their start positions.\n",
    "        is_impossible: False by default, set to True if the example has no possible answer.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        qas_id,\n",
    "        question_text,\n",
    "        context_text,\n",
    "        answer_text,\n",
    "        start_position_character,\n",
    "        title,\n",
    "        answers=[],\n",
    "        is_impossible=False,\n",
    "    ):\n",
    "        self.qas_id = qas_id\n",
    "        self.question_text = question_text\n",
    "        self.context_text = context_text\n",
    "        self.answer_text = answer_text\n",
    "        self.title = title\n",
    "        self.is_impossible = is_impossible\n",
    "        self.answers = answers\n",
    "\n",
    "        self.start_position, self.end_position = 0, 0\n",
    "\n",
    "        doc_tokens = []\n",
    "        char_to_word_offset = []\n",
    "        prev_is_whitespace = True\n",
    "\n",
    "        # Split on whitespace so that different tokens may be attributed to their original position.\n",
    "        for c in self.context_text:\n",
    "            if _is_whitespace(c):\n",
    "                prev_is_whitespace = True\n",
    "            else:\n",
    "                if prev_is_whitespace:\n",
    "                    doc_tokens.append(c)\n",
    "                else:\n",
    "                    doc_tokens[-1] += c\n",
    "                prev_is_whitespace = False\n",
    "            char_to_word_offset.append(len(doc_tokens) - 1)\n",
    "\n",
    "        self.doc_tokens = doc_tokens\n",
    "        self.char_to_word_offset = char_to_word_offset\n",
    "\n",
    "        # Start and end positions only has a value during evaluation.\n",
    "        if start_position_character is not None and not is_impossible:\n",
    "            self.start_position = char_to_word_offset[start_position_character]\n",
    "            self.end_position = char_to_word_offset[\n",
    "                min(start_position_character + len(answer_text) - 1, len(char_to_word_offset) - 1)\n",
    "            ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "interesting-longitude",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46 49\n"
     ]
    }
   ],
   "source": [
    "# 예시로 저희 2주차 강의 예제 가져와볼게요\n",
    "qas_id = \"6521755-0-0\"\n",
    "q_text = \"미국 군대 내 두번째로 높은 직위는 무엇인가?\"\n",
    "a_text = \"미국 육군 부참모 총장\"\n",
    "start_position = 204\n",
    "c_text = \"알렉산더 메이그스 헤이그 2세(영어: Alexander Meigs Haig, Jr., 1924년 12월 2일 ~ 2010년 2월 20일)는 미국의 국무 장관을 지낸 미국의 군인, 관료 및 정치인이다. 로널드 레이건 대통령 밑에서 국무장관을 지냈으며, 리처드 닉슨과 제럴드 포드 대통령 밑에서 백악관 비서실장을 지냈다. 또한 그는 미국 군대에서 2번째로 높은 직위인 미국 육군 부참모 총장과 나토 및 미국 군대의 유럽연합군 최고사령관이었다. 한국 전쟁 시절 더글러스 맥아더 유엔군 사령관의 참모로 직접 참전하였으며, 로널드 레이건 정부 출범당시 초대 국무장관직을 맡아 1980년대 대한민국과 미국의 관계를 조율해 왔다. 저서로 회고록 《경고:현실주의, 레이건과 외교 정책》(1984년 발간)이 있다.\"\n",
    "title = \"알렉산더_헤이그\"\n",
    "\n",
    "example = SquadExample(qas_id, q_text, c_text, a_text, start_position, title)\n",
    "print(example.start_position, example.end_position)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "violent-concept",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cleaend_answer_text, 미국 육군 부참모 총장\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "이건 squad_convert_examples_to_features 에서 각 exmaple 을 처리하는 \n",
    "squad_convert_example_to_features 함수 코드 입니다\n",
    "\n",
    "\"\"\"\n",
    "is_training = True\n",
    "\n",
    "features = []\n",
    "if is_training and not example.is_impossible:\n",
    "    # Get start and end position\n",
    "    start_position = example.start_position\n",
    "    end_position = example.end_position\n",
    "\n",
    "    # If the answer cannot be found in the text, then skip this example.\n",
    "    actual_text = \" \".join(example.doc_tokens[start_position : (end_position + 1)])\n",
    "    cleaned_answer_text = \" \".join(whitespace_tokenize(example.answer_text))\n",
    "    if actual_text.find(cleaned_answer_text) == -1:\n",
    "        print(\"Could not find answer: '%s' vs. '%s'\", actual_text, cleaned_answer_text)\n",
    "        \n",
    "print(f\"cleaend_answer_text, {cleaned_answer_text}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bound-russell",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "tokenizer 을 선언하고 공백기준 토큰들을 subword로 토큰화합니다\n",
    "\"\"\"\n",
    "from transformers import BertTokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-multilingual-cased\")\n",
    "\n",
    "tok_to_orig_index = []\n",
    "orig_to_tok_index = []\n",
    "all_doc_tokens = []\n",
    "for (i, token) in enumerate(example.doc_tokens):\n",
    "    orig_to_tok_index.append(len(all_doc_tokens))\n",
    "    if tokenizer.__class__.__name__ in [\n",
    "        \"RobertaTokenizer\",\n",
    "        \"LongformerTokenizer\",\n",
    "        \"BartTokenizer\",\n",
    "        \"RobertaTokenizerFast\",\n",
    "        \"LongformerTokenizerFast\",\n",
    "        \"BartTokenizerFast\",\n",
    "    ]:\n",
    "        sub_tokens = tokenizer.tokenize(token, add_prefix_space=True)\n",
    "    else:\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "    for sub_token in sub_tokens:\n",
    "        tok_to_orig_index.append(i)\n",
    "        all_doc_tokens.append(sub_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "white-illustration",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "subword 로 토큰을 바꾼 상태에서 start end position 을 맞추는 과정입니다\n",
    "\"\"\"\n",
    "if is_training and not example.is_impossible:\n",
    "    tok_start_position = orig_to_tok_index[example.start_position]\n",
    "    if example.end_position < len(example.doc_tokens) - 1:\n",
    "        tok_end_position = orig_to_tok_index[example.end_position + 1] - 1\n",
    "    else:\n",
    "        tok_end_position = len(all_doc_tokens) - 1\n",
    "\n",
    "    (tok_start_position, tok_end_position) = _improve_answer_span(\n",
    "        all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.answer_text\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cleared-conducting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "225"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_doc_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "spiritual-sentence",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "쿼리를 최대 길이에 맞게 자릅니다\n",
    "\n",
    "sep 토큰을 넣고 config 에서 미리 설정한 전체 길이에 맞게 길이를 조정하도록 준비합니다\n",
    "(sequence_pair_added_tokens, sequence_added_tokens)이 길이 조정 용도입니다\n",
    "\"\"\"\n",
    "spans = []\n",
    "max_query_length = 32\n",
    "# Store the tokenizers which insert 2 separators tokens\n",
    "MULTI_SEP_TOKENS_TOKENIZERS_SET = {\"roberta\", \"camembert\", \"bart\", \"mpnet\"}\n",
    "\n",
    "truncated_query = tokenizer.encode(\n",
    "    example.question_text, add_special_tokens=False, truncation=True, max_length=max_query_length\n",
    ")\n",
    "\n",
    "# Tokenizers who insert 2 SEP tokens in-between <context> & <question> need to have special handling\n",
    "# in the way they compute mask of added tokens.\n",
    "tokenizer_type = type(tokenizer).__name__.replace(\"Tokenizer\", \"\").lower()\n",
    "sequence_added_tokens = (\n",
    "    tokenizer.model_max_length - tokenizer.max_len_single_sentence + 1\n",
    "    if tokenizer_type in MULTI_SEP_TOKENS_TOKENIZERS_SET\n",
    "    else tokenizer.model_max_length - tokenizer.max_len_single_sentence\n",
    ")\n",
    "sequence_pair_added_tokens = tokenizer.model_max_length - tokenizer.max_len_sentences_pair\n",
    "\n",
    "span_doc_tokens = all_doc_tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "stylish-federal",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "overflow\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "미리 지정해둔 doc_stride 변수에 맞게 context 문서를 잘라냅니다\n",
    "즉 하나의 context는 doc_stride 에 맞게\n",
    "여러개의 sub_contexts으로 나뉩니다.\n",
    "여기서는 answer 이 안들어가도 상관하지 않습니다 \n",
    "\n",
    "\"\"\"\n",
    "from transformers.tokenization_utils_base import TruncationStrategy\n",
    "doc_stride= 64\n",
    "padding_strategy = \"max_length\"\n",
    "max_seq_length = 128\n",
    "\n",
    "while len(spans) * doc_stride < len(all_doc_tokens):\n",
    "\n",
    "    # Define the side we want to truncate / pad and the text/pair sorting\n",
    "    if tokenizer.padding_side == \"right\":\n",
    "        texts = truncated_query\n",
    "        pairs = span_doc_tokens\n",
    "        truncation = TruncationStrategy.ONLY_SECOND.value\n",
    "    else:\n",
    "        texts = span_doc_tokens\n",
    "        pairs = truncated_query\n",
    "        truncation = TruncationStrategy.ONLY_FIRST.value\n",
    "\n",
    "    encoded_dict = tokenizer.encode_plus(  # TODO(thom) update this logic\n",
    "        texts,\n",
    "        pairs,\n",
    "        truncation=truncation,\n",
    "        padding=padding_strategy,\n",
    "        max_length=max_seq_length,\n",
    "        return_overflowing_tokens=True,\n",
    "        stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens,\n",
    "        return_token_type_ids=True,\n",
    "    )\n",
    "\n",
    "    paragraph_len = min(\n",
    "        len(all_doc_tokens) - len(spans) * doc_stride,\n",
    "        max_seq_length - len(truncated_query) - sequence_pair_added_tokens,\n",
    "    )\n",
    "\n",
    "    if tokenizer.pad_token_id in encoded_dict[\"input_ids\"]:\n",
    "        if tokenizer.padding_side == \"right\":\n",
    "            non_padded_ids = encoded_dict[\"input_ids\"][: encoded_dict[\"input_ids\"].index(tokenizer.pad_token_id)]\n",
    "        else:\n",
    "            last_padding_id_position = (\n",
    "                len(encoded_dict[\"input_ids\"]) - 1 - encoded_dict[\"input_ids\"][::-1].index(tokenizer.pad_token_id)\n",
    "            )\n",
    "            non_padded_ids = encoded_dict[\"input_ids\"][last_padding_id_position + 1 :]\n",
    "\n",
    "    else:\n",
    "        non_padded_ids = encoded_dict[\"input_ids\"]\n",
    "\n",
    "    tokens = tokenizer.convert_ids_to_tokens(non_padded_ids)\n",
    "\n",
    "    token_to_orig_map = {}\n",
    "    for i in range(paragraph_len):\n",
    "        index = len(truncated_query) + sequence_added_tokens + i if tokenizer.padding_side == \"right\" else i\n",
    "        token_to_orig_map[index] = tok_to_orig_index[len(spans) * doc_stride + i]\n",
    "\n",
    "    encoded_dict[\"paragraph_len\"] = paragraph_len\n",
    "    encoded_dict[\"tokens\"] = tokens\n",
    "    encoded_dict[\"token_to_orig_map\"] = token_to_orig_map\n",
    "    encoded_dict[\"truncated_query_with_special_tokens_length\"] = len(truncated_query) + sequence_added_tokens\n",
    "    encoded_dict[\"token_is_max_context\"] = {}\n",
    "    encoded_dict[\"start\"] = len(spans) * doc_stride\n",
    "    encoded_dict[\"length\"] = paragraph_len\n",
    "\n",
    "    spans.append(encoded_dict)\n",
    "\n",
    "    if \"overflowing_tokens\" not in encoded_dict or (\n",
    "        \"overflowing_tokens\" in encoded_dict and len(encoded_dict[\"overflowing_tokens\"]) == 0\n",
    "    ):\n",
    "        print(\"overflow\")\n",
    "        break\n",
    "        \n",
    "    span_doc_tokens = encoded_dict[\"overflowing_tokens\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cross-bradford",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(spans) # 3 개의 여러개의 subcontexts 으로 나뉩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "restricted-hughes",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "체크 안한 부분 입니다\n",
    "\"\"\"\n",
    "for doc_span_index in range(len(spans)):\n",
    "    for j in range(spans[doc_span_index][\"paragraph_len\"]):\n",
    "        is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j)\n",
    "        index = (\n",
    "            j\n",
    "            if tokenizer.padding_side == \"left\"\n",
    "            else spans[doc_span_index][\"truncated_query_with_special_tokens_length\"] + j\n",
    "        )\n",
    "        spans[doc_span_index][\"token_is_max_context\"][index] = is_max_context\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "declared-surface",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'SquadFeatures' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-84ba3a5cb107>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     features.append(\n\u001b[0;32m---> 55\u001b[0;31m         SquadFeatures(\n\u001b[0m\u001b[1;32m     56\u001b[0m             \u001b[0mspan\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"input_ids\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0mspan\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"attention_mask\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'SquadFeatures' is not defined"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "여러개의 sub_context을 보면서 answer 이 들어가 있는지\n",
    "등의 여부를 확인해 최종적인 feature 를 만들어냅니다\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "for span in spans:\n",
    "    # Identify the position of the CLS token\n",
    "    cls_index = span[\"input_ids\"].index(tokenizer.cls_token_id)\n",
    "\n",
    "    # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer)\n",
    "    # Original TF implem also keep the classification token (set to 0)\n",
    "    p_mask = np.ones_like(span[\"token_type_ids\"])\n",
    "    if tokenizer.padding_side == \"right\":\n",
    "        p_mask[len(truncated_query) + sequence_added_tokens :] = 0\n",
    "    else:\n",
    "        p_mask[-len(span[\"tokens\"]) : -(len(truncated_query) + sequence_added_tokens)] = 0\n",
    "\n",
    "    pad_token_indices = np.where(span[\"input_ids\"] == tokenizer.pad_token_id)\n",
    "    special_token_indices = np.asarray(\n",
    "        tokenizer.get_special_tokens_mask(span[\"input_ids\"], already_has_special_tokens=True)\n",
    "    ).nonzero()\n",
    "\n",
    "    p_mask[pad_token_indices] = 1\n",
    "    p_mask[special_token_indices] = 1\n",
    "\n",
    "    # Set the cls index to 0: the CLS index can be used for impossible answers\n",
    "    p_mask[cls_index] = 0\n",
    "\n",
    "    span_is_impossible = example.is_impossible\n",
    "    start_position = 0\n",
    "    end_position = 0\n",
    "    if is_training and not span_is_impossible:\n",
    "        # For training, if our document chunk does not contain an annotation\n",
    "        # we throw it out, since there is nothing to predict.\n",
    "        doc_start = span[\"start\"]\n",
    "        doc_end = span[\"start\"] + span[\"length\"] - 1\n",
    "        out_of_span = False\n",
    "\n",
    "        if not (tok_start_position >= doc_start and tok_end_position <= doc_end):\n",
    "            out_of_span = True\n",
    "\n",
    "        if out_of_span:\n",
    "            start_position = cls_index\n",
    "            end_position = cls_index\n",
    "            span_is_impossible = True\n",
    "        else:\n",
    "            if tokenizer.padding_side == \"left\":\n",
    "                doc_offset = 0\n",
    "            else:\n",
    "                doc_offset = len(truncated_query) + sequence_added_tokens\n",
    "\n",
    "            start_position = tok_start_position - doc_start + doc_offset\n",
    "            end_position = tok_end_position - doc_start + doc_offset\n",
    "\n",
    "    features.append(\n",
    "        SquadFeatures(\n",
    "            span[\"input_ids\"],\n",
    "            span[\"attention_mask\"],\n",
    "            span[\"token_type_ids\"],\n",
    "            cls_index,\n",
    "            p_mask.tolist(),\n",
    "            example_index=0,  # Can not set unique_id and example_index here. They will be set after multiple processing.\n",
    "            unique_id=0,\n",
    "            paragraph_len=span[\"paragraph_len\"],\n",
    "            token_is_max_context=span[\"token_is_max_context\"],\n",
    "            tokens=span[\"tokens\"],\n",
    "            token_to_orig_map=span[\"token_to_orig_map\"],\n",
    "            start_position=start_position,\n",
    "            end_position=end_position,\n",
    "            is_impossible=span_is_impossible,\n",
    "            qas_id=example.qas_id,\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sensitive-median",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "martial-costs",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hydraulic-transfer",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}