chatbot_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1e4170a9",
   "metadata": {},
   "source": [
    "# 프로젝트: 한국어 데이터로 챗봇 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de385725",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b7b688",
   "metadata": {},
   "source": [
    "## Step 1. 데이터 수집하기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4aede5c2",
   "metadata": {},
   "source": [
    "### 포지셔널 행렬구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "13dc4c66",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 포지셔널 인코딩 레이어\n",
    "class PositionalEncoding(tf.keras.layers.Layer):\n",
    "\n",
    "  def __init__(self, position, d_model):\n",
    "    super(PositionalEncoding, self).__init__()\n",
    "    self.pos_encoding = self.positional_encoding(position, d_model)\n",
    "\n",
    "  def get_angles(self, position, i, d_model):\n",
    "    angles = 1 / tf.pow(10000, (2 * (i // 2)) / tf.cast(d_model, tf.float32))\n",
    "    return position * angles\n",
    "\n",
    "  def positional_encoding(self, position, d_model):\n",
    "    # 각도 배열 생성\n",
    "    angle_rads = self.get_angles(\n",
    "        position=tf.range(position, dtype=tf.float32)[:, tf.newaxis],\n",
    "        i=tf.range(d_model, dtype=tf.float32)[tf.newaxis, :],\n",
    "        d_model=d_model)\n",
    "\n",
    "    # 배열의 짝수 인덱스에는 sin 함수 적용\n",
    "    sines = tf.math.sin(angle_rads[:, 0::2])\n",
    "    # 배열의 홀수 인덱스에는 cosine 함수 적용\n",
    "    cosines = tf.math.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    # sin과 cosine이 교차되도록 재배열\n",
    "    pos_encoding = tf.stack([sines, cosines], axis=0)\n",
    "    pos_encoding = tf.transpose(pos_encoding,[1, 2, 0]) \n",
    "    pos_encoding = tf.reshape(pos_encoding, [position, d_model])\n",
    "\n",
    "    pos_encoding = pos_encoding[tf.newaxis, ...]\n",
    "    return tf.cast(pos_encoding, tf.float32)\n",
    "\n",
    "  def call(self, inputs):\n",
    "    return inputs + self.pos_encoding[:, :tf.shape(inputs)[1], :]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b965b0c7",
   "metadata": {},
   "source": [
    "### 스케일드 닷 프로덕트 어텐션 구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cc0935e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 스케일드 닷 프로덕트 어텐션 함수\n",
    "def scaled_dot_product_attention(query, key, value, mask):\n",
    "  # 어텐션 가중치는 Q와 K의 닷 프로덕트\n",
    "  matmul_qk = tf.matmul(query, key, transpose_b=True)\n",
    "\n",
    "  # 가중치를 정규화\n",
    "  depth = tf.cast(tf.shape(key)[-1], tf.float32)\n",
    "  logits = matmul_qk / tf.math.sqrt(depth)\n",
    "\n",
    "  # 패딩에 마스크 추가\n",
    "  if mask is not None:\n",
    "    logits += (mask * -1e9)\n",
    "\n",
    "  # softmax적용\n",
    "  attention_weights = tf.nn.softmax(logits, axis=-1)\n",
    "\n",
    "  # 최종 어텐션은 가중치와 V의 닷 프로덕트\n",
    "  output = tf.matmul(attention_weights, value)\n",
    "  return output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a45ea56e",
   "metadata": {},
   "source": [
    "### 멀티 헤드 어텐션 구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "da164aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "\n",
    "  def __init__(self, d_model, num_heads, name=\"multi_head_attention\"):\n",
    "    super(MultiHeadAttention, self).__init__(name=name)\n",
    "    self.num_heads = num_heads\n",
    "    self.d_model = d_model\n",
    "\n",
    "    assert d_model % self.num_heads == 0\n",
    "\n",
    "    self.depth = d_model // self.num_heads\n",
    "\n",
    "    self.query_dense = tf.keras.layers.Dense(units=d_model)\n",
    "    self.key_dense = tf.keras.layers.Dense(units=d_model)\n",
    "    self.value_dense = tf.keras.layers.Dense(units=d_model)\n",
    "\n",
    "    self.dense = tf.keras.layers.Dense(units=d_model)\n",
    "\n",
    "  def split_heads(self, inputs, batch_size):\n",
    "    inputs = tf.reshape(\n",
    "        inputs, shape=(batch_size, -1, self.num_heads, self.depth))\n",
    "    return tf.transpose(inputs, perm=[0, 2, 1, 3])\n",
    "\n",
    "  def call(self, inputs):\n",
    "    query, key, value, mask = inputs['query'], inputs['key'], inputs[\n",
    "        'value'], inputs['mask']\n",
    "    batch_size = tf.shape(query)[0]\n",
    "\n",
    "    # Q, K, V에 각각 Dense를 적용합니다\n",
    "    query = self.query_dense(query) # [[YOUR CODE]]\n",
    "    key = self.key_dense(key) # [[YOUR CODE]]\n",
    "    value = self.value_dense(value) # [[YOUR CODE]]\n",
    "\n",
    "    # 병렬 연산을 위한 머리를 여러 개 만듭니다\n",
    "    query = self.split_heads(query, batch_size) # [[YOUR CODE]]\n",
    "    key = self.split_heads(key, batch_size) # [[YOUR CODE]]\n",
    "    value = self.split_heads(value, batch_size) # [[YOUR CODE]]\n",
    "\n",
    "    # 스케일드 닷 프로덕트 어텐션 함수\n",
    "    scaled_attention = scaled_dot_product_attention(query, key, value, mask)\n",
    "\n",
    "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
    "\n",
    "    # 어텐션 연산 후에 각 결과를 다시 연결(concatenate)합니다\n",
    "    concat_attention = tf.reshape(scaled_attention,\n",
    "                                  (batch_size, -1, self.d_model))\n",
    "\n",
    "    # 최종 결과에도 Dense를 한 번 더 적용합니다\n",
    "    outputs = self.dense(concat_attention)\n",
    "\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c2c249c",
   "metadata": {},
   "source": [
    "### 패딩 마스킹 구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1c5ca63e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_padding_mask(x):\n",
    "  mask = tf.cast(tf.math.equal(x, 0), tf.float32)\n",
    "  # (batch_size, 1, 1, sequence length)\n",
    "  return mask[:, tf.newaxis, tf.newaxis, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "131867cb",
   "metadata": {},
   "source": [
    "#### '0\"색 부분을 마스킹 함수로 구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "83fef71d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_look_ahead_mask(x):\n",
    "  seq_len = tf.shape(x)[1]\n",
    "  look_ahead_mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)\n",
    "  padding_mask = create_padding_mask(x)\n",
    "  return tf.maximum(look_ahead_mask, padding_mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c07773a",
   "metadata": {},
   "source": [
    "### 인코더 구현"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4733aec",
   "metadata": {},
   "source": [
    "#### - 인코더 층 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0f3cb7b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 인코더 하나의 레이어를 함수로 구현.\n",
    "# 이 하나의 레이어 안에는 두 개의 서브 레이어가 존재합니다.\n",
    "def encoder_layer(units, d_model, num_heads, dropout, name=\"encoder_layer\"):\n",
    "  inputs = tf.keras.Input(shape=(None, d_model), name=\"inputs\")\n",
    "\n",
    "  # 패딩 마스크 사용\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
    "\n",
    "  # 첫 번째 서브 레이어 : 멀티 헤드 어텐션 수행 (셀프 어텐션)\n",
    "  attention = MultiHeadAttention(\n",
    "      d_model, num_heads, name=\"attention\")({\n",
    "          'query': inputs,\n",
    "          'key': inputs,\n",
    "          'value': inputs,\n",
    "          'mask': padding_mask\n",
    "      })\n",
    "\n",
    "  # 어텐션의 결과는 Dropout과 Layer Normalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  attention = tf.keras.layers.Dropout(rate=dropout)(attention)\n",
    "  attention = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(inputs + attention)\n",
    "\n",
    "  # 두 번째 서브 레이어 : 2개의 완전연결층\n",
    "  outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention)\n",
    "  outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
    "\n",
    "  # 완전연결층의 결과는 Dropout과 LayerNormalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
    "  outputs = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(attention + outputs)\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, padding_mask], outputs=outputs, name=name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ec5f44b",
   "metadata": {},
   "source": [
    "#### - 인코더 층을 쌓아 인코더 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "767e6dfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def encoder(vocab_size,\n",
    "            num_layers,\n",
    "            units,\n",
    "            d_model,\n",
    "            num_heads,\n",
    "            dropout,\n",
    "            name=\"encoder\"):\n",
    "  inputs = tf.keras.Input(shape=(None,), name=\"inputs\")\n",
    "\n",
    "  # 패딩 마스크 사용\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name=\"padding_mask\")\n",
    "\n",
    "  # 임베딩 레이어\n",
    "  embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)\n",
    "  embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
    "\n",
    "  # 포지셔널 인코딩\n",
    "  embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)\n",
    "\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)\n",
    "\n",
    "  # num_layers만큼 쌓아올린 인코더의 층.\n",
    "  for i in range(num_layers):\n",
    "    outputs = encoder_layer(\n",
    "        units=units,\n",
    "        d_model=d_model,\n",
    "        num_heads=num_heads,\n",
    "        dropout=dropout,\n",
    "        name=\"encoder_layer_{}\".format(i),\n",
    "    )([outputs, padding_mask])\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, padding_mask], outputs=outputs, name=name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecc8c1c5",
   "metadata": {},
   "source": [
    "### 디코더 구현"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "152dcee9",
   "metadata": {},
   "source": [
    "#### - 디코더층 구현(세 개의 서브 층을 내부적으로 구현)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9437ad3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 디코더 하나의 레이어를 함수로 구현.\n",
    "# 이 하나의 레이어 안에는 세 개의 서브 레이어가 존재합니다.\n",
    "def decoder_layer(units, d_model, num_heads, dropout, name=\"decoder_layer\"):\n",
    "  inputs = tf.keras.Input(shape=(None, d_model), name=\"inputs\")\n",
    "  enc_outputs = tf.keras.Input(shape=(None, d_model), name=\"encoder_outputs\")\n",
    "  look_ahead_mask = tf.keras.Input(\n",
    "      shape=(1, None, None), name=\"look_ahead_mask\")\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')\n",
    "\n",
    "  # 첫 번째 서브 레이어 : 멀티 헤드 어텐션 수행 (셀프 어텐션)\n",
    "  attention1 = MultiHeadAttention(\n",
    "      d_model, num_heads, name=\"attention_1\")(inputs={\n",
    "          'query': inputs,\n",
    "          'key': inputs,\n",
    "          'value': inputs,\n",
    "          'mask': look_ahead_mask\n",
    "      })\n",
    "\n",
    "  # 멀티 헤드 어텐션의 결과는 LayerNormalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  attention1 = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(attention1 + inputs)\n",
    "\n",
    "  # 두 번째 서브 레이어 : 마스크드 멀티 헤드 어텐션 수행 (인코더-디코더 어텐션)\n",
    "  attention2 = MultiHeadAttention(\n",
    "      d_model, num_heads, name=\"attention_2\")(inputs={\n",
    "          'query': attention1,\n",
    "          'key': enc_outputs,\n",
    "          'value': enc_outputs,\n",
    "          'mask': padding_mask\n",
    "      })\n",
    "\n",
    "  # 마스크드 멀티 헤드 어텐션의 결과는\n",
    "  # Dropout과 LayerNormalization이라는 훈련을 돕는 테크닉을 수행\n",
    "  attention2 = tf.keras.layers.Dropout(rate=dropout)(attention2)\n",
    "  attention2 = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(attention2 + attention1)\n",
    "\n",
    "  # 세 번째 서브 레이어 : 2개의 완전연결층\n",
    "  outputs = tf.keras.layers.Dense(units=units, activation='relu')(attention2)\n",
    "  outputs = tf.keras.layers.Dense(units=d_model)(outputs)\n",
    "\n",
    "  # 완전연결층의 결과는 Dropout과 LayerNormalization 수행\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(outputs)\n",
    "  outputs = tf.keras.layers.LayerNormalization(\n",
    "      epsilon=1e-6)(outputs + attention2)\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],\n",
    "      outputs=outputs,\n",
    "      name=name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b681b15c",
   "metadata": {},
   "source": [
    "#### - 디코더 층을 쌓아 디코더 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2b3bc80b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoder(vocab_size,\n",
    "            num_layers,\n",
    "            units,\n",
    "            d_model,\n",
    "            num_heads,\n",
    "            dropout,\n",
    "            name='decoder'):\n",
    "  inputs = tf.keras.Input(shape=(None,), name='inputs')\n",
    "  enc_outputs = tf.keras.Input(shape=(None, d_model), name='encoder_outputs')\n",
    "  look_ahead_mask = tf.keras.Input(\n",
    "      shape=(1, None, None), name='look_ahead_mask')\n",
    "\n",
    "  # 패딩 마스크\n",
    "  padding_mask = tf.keras.Input(shape=(1, 1, None), name='padding_mask')\n",
    "  \n",
    "  # 임베딩 레이어\n",
    "  embeddings = tf.keras.layers.Embedding(vocab_size, d_model)(inputs)\n",
    "  embeddings *= tf.math.sqrt(tf.cast(d_model, tf.float32))\n",
    "\n",
    "  # 포지셔널 인코딩\n",
    "  embeddings = PositionalEncoding(vocab_size, d_model)(embeddings)\n",
    "\n",
    "  # Dropout이라는 훈련을 돕는 테크닉을 수행\n",
    "  outputs = tf.keras.layers.Dropout(rate=dropout)(embeddings)\n",
    "\n",
    "  for i in range(num_layers):\n",
    "    outputs = decoder_layer(\n",
    "        units=units,\n",
    "        d_model=d_model,\n",
    "        num_heads=num_heads,\n",
    "        dropout=dropout,\n",
    "        name='decoder_layer_{}'.format(i),\n",
    "    )(inputs=[outputs, enc_outputs, look_ahead_mask, padding_mask])\n",
    "\n",
    "  return tf.keras.Model(\n",
    "      inputs=[inputs, enc_outputs, look_ahead_mask, padding_mask],\n",
    "      outputs=outputs,\n",
    "      name=name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0ed96c2",
   "metadata": {},
   "source": [
    "## Step 2. 데이터 전처리하기 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "90612c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 전처리 함수\n",
    "def preprocess_sentence(sentence):\n",
    "  # 입력받은 sentence를 소문자로 변경하고 양쪽 공백을 제거\n",
    "    sentence = sentence.strip().lower()  \n",
    "\n",
    "    # 단어와 구두점(punctuation) 사이의 거리를 만듦\n",
    "    sentence = re.sub(r\"([?.!,])\", r\" \\1 \", sentence)\n",
    "    sentence = re.sub(r'[\" \"]+', \" \", sentence)\n",
    "\n",
    "    # (a-z, A-Z, 숫자,  한글, \".\", \"?\", \"!\", \",\")를 제외한 모든 문자를 공백(' ')으로 대체\n",
    "    sentence = re.sub(r\"[^a-zA-Z0-9ㄱ-ㅎ가-힣\\.\\?\\!,]\", \" \", sentence)  \n",
    "\n",
    "    sentence = sentence.strip()\n",
    "    \n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f420fd9",
   "metadata": {},
   "source": [
    "데이터를 로드하는 동시에 전처리 함수를 호출하여 질문과 답변의 쌍을 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4165fa1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_conversations():\n",
    "  id2line = {}\n",
    "  inputs, outputs = [], []\n",
    "    \n",
    "  with open(\"ChatbotData.csv\", errors='ignore') as file:\n",
    "    lines = file.readlines()\n",
    "\n",
    "  for line in lines:\n",
    "    parts = line.replace('\\n', '').split(',')\n",
    "    inputs.append(preprocess_sentence(parts[0]))\n",
    "    outputs.append(preprocess_sentence(parts[1]))\n",
    "        \n",
    "  return inputs, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9a132d9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "전체 샘플 수 : 11824\n",
      "전체 샘플 수 : 11824\n"
     ]
    }
   ],
   "source": [
    "# 데이터를 로드하고 전처리하여 질문을 questions, 답변을 answers에 저장합니다.\n",
    "questions, answers = load_conversations()\n",
    "print('전체 샘플 수 :', len(questions))\n",
    "print('전체 샘플 수 :', len(answers))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41835fe1",
   "metadata": {},
   "source": [
    "## Step 3. SubwordTextEncoder 사용하기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efea77f2",
   "metadata": {},
   "source": [
    "### 1. 단어장(Vocabulary) 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b66dad95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "살짝 오래 걸릴 수 있어요. 스트레칭 한 번 해볼까요? 👐\n"
     ]
    }
   ],
   "source": [
    "# 질문과 답변 데이터셋에 대해서 Vocabulary 생성\n",
    "tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(questions + answers, target_vocab_size=2**13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a71cfdb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시작 토큰과 종료 토큰에 고유한 정수를 부여합니다.\n",
    "START_TOKEN, END_TOKEN = [tokenizer.vocab_size], [tokenizer.vocab_size + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7564ce71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START_TOKEN의 번호 : [8143]\n",
      "END_TOKEN의 번호 : [8144]\n"
     ]
    }
   ],
   "source": [
    "print('START_TOKEN의 번호 :' ,[tokenizer.vocab_size])\n",
    "print('END_TOKEN의 번호 :' ,[tokenizer.vocab_size + 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f9f8aa22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8145\n"
     ]
    }
   ],
   "source": [
    "# 시작 토큰과 종료 토큰을 고려하여 +2를 하여 단어장의 크기를 산정합니다.\n",
    "VOCAB_SIZE = tokenizer.vocab_size + 2\n",
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b267fd45",
   "metadata": {},
   "source": [
    "### 2. 각 단어를 고유한 정수로 인코딩(Integer encoding) & 패딩(Padding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "513e35d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "# 샘플의 최대 허용 길이 또는 패딩 후의 최종 길이\n",
    "MAX_LENGTH = 40\n",
    "print(MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7d87dd79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 정수 인코딩, 최대 길이를 초과하는 샘플 제거, 패딩\n",
    "def tokenize_and_filter(inputs, outputs):\n",
    "  tokenized_inputs, tokenized_outputs = [], []\n",
    "  \n",
    "  for (sentence1, sentence2) in zip(inputs, outputs):\n",
    "    # 정수 인코딩 과정에서 시작 토큰과 종료 토큰을 추가\n",
    "    sentence1 = START_TOKEN + tokenizer.encode(sentence1) + END_TOKEN\n",
    "    sentence2 = START_TOKEN + tokenizer.encode(sentence2) + END_TOKEN\n",
    "\n",
    "    # 최대 길이 40 이하인 경우에만 데이터셋으로 허용\n",
    "    if len(sentence1) <= MAX_LENGTH and len(sentence2) <= MAX_LENGTH:\n",
    "      tokenized_inputs.append(sentence1)\n",
    "      tokenized_outputs.append(sentence2)\n",
    "  \n",
    "  # 최대 길이 40으로 모든 데이터셋을 패딩\n",
    "  tokenized_inputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "      tokenized_inputs, maxlen=MAX_LENGTH, padding='post')\n",
    "  tokenized_outputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "      tokenized_outputs, maxlen=MAX_LENGTH, padding='post')\n",
    "  \n",
    "  return tokenized_inputs, tokenized_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "885849b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "단어장의 크기 : 8145\n",
      "필터링 후의 질문 샘플 개수: 11824\n",
      "필터링 후의 답변 샘플 개수: 11824\n"
     ]
    }
   ],
   "source": [
    "questions, answers = tokenize_and_filter(questions, answers)\n",
    "print('단어장의 크기 :',(VOCAB_SIZE))\n",
    "print('필터링 후의 질문 샘플 개수: {}'.format(len(questions)))\n",
    "print('필터링 후의 답변 샘플 개수: {}'.format(len(answers)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b201af01",
   "metadata": {},
   "source": [
    "### 3. 교사 강요(Teacher Forcing) 사용하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "58c7767b",
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "BUFFER_SIZE = 20000\n",
    "\n",
    "# 디코더는 이전의 target을 다음의 input으로 사용합니다.\n",
    "# 이에 따라 outputs에서는 START_TOKEN을 제거하겠습니다.\n",
    "dataset = tf.data.Dataset.from_tensor_slices((\n",
    "    {\n",
    "        'inputs': questions,\n",
    "        'dec_inputs': answers[:, :-1]\n",
    "    },\n",
    "    {\n",
    "        'outputs': answers[:, 1:]\n",
    "    },\n",
    "))\n",
    "\n",
    "dataset = dataset.cache()\n",
    "dataset = dataset.shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE)\n",
    "dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33ebf61e",
   "metadata": {},
   "source": [
    "## Step 4. 모델 구성하기"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88d23cca",
   "metadata": {},
   "source": [
    "### 트랜스포머 함수 정의"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a177422e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def transformer(vocab_size,\n",
    "                num_layers,\n",
    "                units,\n",
    "                d_model,\n",
    "                num_heads,\n",
    "                dropout,\n",
    "                name=\"transformer\"):\n",
    "  inputs = tf.keras.Input(shape=(None,), name=\"inputs\")\n",
    "  dec_inputs = tf.keras.Input(shape=(None,), name=\"dec_inputs\")\n",
    "\n",
    "  # 인코더에서 패딩을 위한 마스크\n",
    "  enc_padding_mask = tf.keras.layers.Lambda(\n",
    "      create_padding_mask, output_shape=(1, 1, None),\n",
    "      name='enc_padding_mask')(inputs)\n",
    "\n",
    "  # 디코더에서 미래의 토큰을 마스크 하기 위해서 사용합니다.\n",
    "  # 내부적으로 패딩 마스크도 포함되어져 있습니다.\n",
    "  look_ahead_mask = tf.keras.layers.Lambda(\n",
    "      create_look_ahead_mask,\n",
    "      output_shape=(1, None, None),\n",
    "      name='look_ahead_mask')(dec_inputs)\n",
    "\n",
    "  # 두 번째 어텐션 블록에서 인코더의 벡터들을 마스킹\n",
    "  # 디코더에서 패딩을 위한 마스크\n",
    "  dec_padding_mask = tf.keras.layers.Lambda(\n",
    "      create_padding_mask, output_shape=(1, 1, None),\n",
    "      name='dec_padding_mask')(inputs)\n",
    "\n",
    "  # 인코더\n",
    "  enc_outputs = encoder(\n",
    "      vocab_size=vocab_size,\n",
    "      num_layers=num_layers,\n",
    "      units=units,\n",
    "      d_model=d_model,\n",
    "      num_heads=num_heads,\n",
    "      dropout=dropout,\n",
    "  )(inputs=[inputs, enc_padding_mask])\n",
    "\n",
    "  # 디코더\n",
    "  dec_outputs = decoder(\n",
    "      vocab_size=vocab_size,\n",
    "      num_layers=num_layers,\n",
    "      units=units,\n",
    "      d_model=d_model,\n",
    "      num_heads=num_heads,\n",
    "      dropout=dropout,\n",
    "  )(inputs=[dec_inputs, enc_outputs, look_ahead_mask, dec_padding_mask])\n",
    "\n",
    "  # 완전연결층\n",
    "  outputs = tf.keras.layers.Dense(units=vocab_size, name=\"outputs\")(dec_outputs)\n",
    "\n",
    "  return tf.keras.Model(inputs=[inputs, dec_inputs], outputs=outputs, name=name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e65bb7f",
   "metadata": {},
   "source": [
    "### 1. 모델 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "74439c61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"transformer\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "inputs (InputLayer)             [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dec_inputs (InputLayer)         [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "enc_padding_mask (Lambda)       (None, 1, 1, None)   0           inputs[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "encoder (Functional)            (None, None, 256)    3139328     inputs[0][0]                     \n",
      "                                                                 enc_padding_mask[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "look_ahead_mask (Lambda)        (None, 1, None, None 0           dec_inputs[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dec_padding_mask (Lambda)       (None, 1, 1, None)   0           inputs[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "decoder (Functional)            (None, None, 256)    3666688     dec_inputs[0][0]                 \n",
      "                                                                 encoder[0][0]                    \n",
      "                                                                 look_ahead_mask[0][0]            \n",
      "                                                                 dec_padding_mask[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "outputs (Dense)                 (None, None, 8145)   2093265     decoder[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 8,899,281\n",
      "Trainable params: 8,899,281\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "\n",
    "# 하이퍼파라미터\n",
    "NUM_LAYERS = 2 # 인코더와 디코더의 층의 개수\n",
    "D_MODEL = 256 # 인코더와 디코더 내부의 입, 출력의 고정 차원\n",
    "NUM_HEADS = 8 # 멀티 헤드 어텐션에서의 헤드 수 \n",
    "UNITS = 512 # 피드 포워드 신경망의 은닉층의 크기\n",
    "DROPOUT = 0.1 # 드롭아웃의 비율\n",
    "\n",
    "model = transformer(\n",
    "    vocab_size=VOCAB_SIZE,\n",
    "    num_layers=NUM_LAYERS,\n",
    "    units=UNITS,\n",
    "    d_model=D_MODEL,\n",
    "    num_heads=NUM_HEADS,\n",
    "    dropout=DROPOUT)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a97d3482",
   "metadata": {},
   "source": [
    "### 2. 손실 함수(Loss function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "383151d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_function(y_true, y_pred):\n",
    "  y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))\n",
    "  \n",
    "  loss = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "      from_logits=True, reduction='none')(y_true, y_pred)\n",
    "\n",
    "  mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)\n",
    "  loss = tf.multiply(loss, mask)\n",
    "\n",
    "  return tf.reduce_mean(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b396307c",
   "metadata": {},
   "source": [
    "### 3. 커스텀 된 학습률(Learning rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7b8af72f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "\n",
    "  def __init__(self, d_model, warmup_steps=4000):\n",
    "    super(CustomSchedule, self).__init__()\n",
    "\n",
    "    self.d_model = d_model\n",
    "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "\n",
    "    self.warmup_steps = warmup_steps\n",
    "\n",
    "  def __call__(self, step):\n",
    "    arg1 = tf.math.rsqrt(step)\n",
    "    arg2 = step * (self.warmup_steps**-1.5)\n",
    "\n",
    "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "eb7734e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Train Step')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEGCAYAAABYV4NmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyBElEQVR4nO3deZxcVZ3//9en9+4k3Uk6nZA9gYQlIAg0GVBUBJXgFpcwJsPMoKJ8HWHcZr4OjMv4ZYbvT9SvfNVBEYUBfaABUb9EjUaGRRGB0MiaQKBJAknIvnRn6+qu7s/vj3uqU2mququr6/ZW7+fjUY++de65556qdO6nz3LPNXdHRESk0EqGugIiIjI6KcCIiEgsFGBERCQWCjAiIhILBRgREYlF2VBXYChNmjTJ58yZM9TVEBEZUR5//PFd7t7QV76iDjBz5syhqalpqKshIjKimNnLueRTF5mIiMRCAUZERGKhACMiIrFQgBERkVgowIiISCxiDTBmtsjM1plZs5ldlWF/pZndEfY/amZz0vZdHdLXmdmFaem3mNkOM3s2yzn/yczczCbF8qFERCQnsQUYMysFbgAuAhYAy8xsQY9slwF73X0ecD1wXTh2AbAUOBlYBHw3lAdwa0jLdM6ZwDuAVwr6YUREpN/ibMEsBJrdfb27twPLgcU98iwGbgvbdwEXmJmF9OXunnD3DUBzKA93/yOwJ8s5rwc+DwzJMwi2t7bx+zXbhuLUIiLDTpwBZjqwKe395pCWMY+7J4EWoD7HY49iZouBLe7+VB/5LjezJjNr2rlzZy6fI2d/+8NHufzHj5NIdha0XBGRkWhUDPKbWQ3wr8CX+8rr7je5e6O7NzY09LnSQb9s3nsYgNbDyYKWKyIyEsUZYLYAM9PezwhpGfOYWRlQB+zO8dh0xwFzgafMbGPI/xczO2YA9e+36opomKjlcMdgnlZEZFiKM8A8Bsw3s7lmVkE0aL+iR54VwKVhewlwn0fPcF4BLA2zzOYC84HV2U7k7s+4+2R3n+Puc4i61M5w90EdEKkuTwWY9sE8rYjIsBRbgAljKlcCq4DngDvdfY2ZXWNm7w3ZbgbqzawZ+BxwVTh2DXAnsBb4HXCFu3cCmNlPgYeBE8xss5ldFtdn6K9UC2bfIbVgRERiXU3Z3VcCK3ukfTltuw24OMux1wLXZkhflsN55/S3roWQasEowIiIjJJB/uGiO8BoDEZERAGmkCrKoq+z5ZDGYEREFGAKqL2zC1ALRkQEFGAKKpEMAUZjMCIiCjCFlOiI7uBXC0ZERAGmoFJdZBqDERFRgCmoRIfGYEREUhRgCkhjMCIiRyjAFFBqFeXWtg46u4bkiQEiIsOGAkwBJZJdVJaV4A6t6iYTkSKnAFMg7k57soupdVUA7NFAv4gUOQWYAkmNv0wbXw3Arv2JoayOiMiQU4ApkJ4BZvdBtWBEpLgpwBRIaoB/eqoFc0AtGBEpbgowBdIeWjDH1FVhBrsOqAUjIsVNAaZAUl1kNRWlTKypUAtGRIqeAkyBpO7irywrpX5sBbsVYESkyCnAFEhqDKayvIRJYyvZrS4yESlyCjAFkuoiqywtoX5spbrIRKToxRpgzGyRma0zs2YzuyrD/kozuyPsf9TM5qTtuzqkrzOzC9PSbzGzHWb2bI+yvm5mz5vZ02b2SzMbH+dn66k7wJSXMGlshVowIlL0YgswZlYK3ABcBCwAlpnZgh7ZLgP2uvs84HrgunDsAmApcDKwCPhuKA/g1pDW0z3AKe5+KvACcHVBP1AfUs+CqSwrZdLYSvYnkrSFNBGRYhRnC2Yh0Ozu6929HVgOLO6RZzFwW9i+C7jAzCykL3f3hLtvAJpDebj7H4E9PU/m7r9392R4+wgwo9AfqDfdLZiyEurHVAC62VJEilucAWY6sCnt/eaQljFPCA4tQH2Ox/bmo8BvM+0ws8vNrMnMmnbu3NmPInvXnjwyi6xhXCUAO7VcjIgUsVE3yG9mXwCSwO2Z9rv7Te7e6O6NDQ0NBTtv+hjMlNpowcttLW0FK19EZKSJM8BsAWamvZ8R0jLmMbMyoA7YneOxr2FmHwbeDVzi7oP6QJbuacplJd0rKm9rOTyYVRARGVbiDDCPAfPNbK6ZVRAN2q/okWcFcGnYXgLcFwLDCmBpmGU2F5gPrO7tZGa2CPg88F53P1TAz5GTRFoX2cQxFVSUlrC1VS0YESlesQWYMKZyJbAKeA64093XmNk1ZvbekO1moN7MmoHPAVeFY9cAdwJrgd8BV7h7J4CZ/RR4GDjBzDab2WWhrP8ExgH3mNmTZnZjXJ8tk9Sd/BVlJZgZU+oq2a4uMhEpYmVxFu7uK4GVPdK+nLbdBlyc5dhrgWszpC/Lkn/egCo7QIlkJ2UlRmmJATC1tpqtCjAiUsRG3SD/UEk9LjllSl0V29RFJiJFTAGmQBLJTirLS7vfT62rYltLG4M810BEZNhQgCmQREePFkxtFYlkF/sOdQxhrUREho4CTIG0dx4dYLqnKqubTESKlAJMgUQtmCNdZMfU6WZLESluCjAFEo3BHPk6p9VVA7Bln262FJHipABTID1nkU0eV0lFaQmb9g76PZ8iIsOCAkyBJJJdVKQFmJISY8aEajbtUYARkeKkAFMgiWTnUWMwADMn1rBpj7rIRKQ4KcAUSM9pygAzJ1bzilowIlKkFGAKpOcYDMDMCTW0HO6g5bDuhRGR4qMAUyDtya7XdJHNmlgDoHEYESlKCjAF0nOaMkRjMACbNZNMRIqQAkyBZOwi627BaKBfRIqPAkyBJDJ0kdVVl1NbVcbLew4OUa1ERIaOAkwBJDu76Ozy17RgAOZOGsPGXeoiE5HiowBTAKnHJVdkCDDHTR7LSzsPDHaVRESGnAJMAaQCTKYWzHENY9na0saBRHKwqyUiMqQUYAogkewEOOqBYynHNYwFYL1aMSJSZGINMGa2yMzWmVmzmV2VYX+lmd0R9j9qZnPS9l0d0teZ2YVp6beY2Q4ze7ZHWRPN7B4zezH8nBDnZ0uX6Mjegpk3eQyAuslEpOjEFmDMrBS4AbgIWAAsM7MFPbJdBux193nA9cB14dgFwFLgZGAR8N1QHsCtIa2nq4B73X0+cG94PyjaO1MB5rUtmFkTx1BaYry0QzPJRKS4xNmCWQg0u/t6d28HlgOLe+RZDNwWtu8CLjAzC+nL3T3h7huA5lAe7v5HYE+G86WXdRvwvgJ+ll711oKpKCthdn2NWjAiUnTiDDDTgU1p7zeHtIx53D0JtAD1OR7b0xR33xq2twFTMmUys8vNrMnMmnbu3JnL5+jTkTGYzF/ncQ2aSSYixWdUDvK7uwOeZd9N7t7o7o0NDQ0FOd+RWWSv7SIDmDd5LBt2HaQ95BMRKQZxBpgtwMy09zNCWsY8ZlYG1AG7czy2p+1mNjWUNRXYkXfN+ynVgsl0HwzASVNr6eh0tWJEpKjEGWAeA+ab2VwzqyAatF/RI88K4NKwvQS4L7Q+VgBLwyyzucB8YHUf50sv61Lg7gJ8hpz0NgYDsGDqOADWvto6WFUSERlysQWYMKZyJbAKeA64093XmNk1ZvbekO1moN7MmoHPEWZ+ufsa4E5gLfA74Ap37wQws58CDwMnmNlmM7sslPVV4O1m9iLwtvB+UPR2oyXA3EljqSovYe1WBRgRKR5lcRbu7iuBlT3Svpy23QZcnOXYa4FrM6Qvy5J/N3DBQOqbr95utAQoLTFOmDKO5xRgRKSIjMpB/sHW3kcLBmDBtFrWbm0l6gEUERn9FGAKoK8uMoAFU2vZd6iDrS1tg1UtEZEhpQBTAH1NU4ZoJhnAGg30i0iRUIApgERHJ2ZQXmpZ8yyYVkuJwdOb9w1exUREhpACTAGkHpccrXKTWU1FGSceU8sTr+wbvIqJiAyhPgOMmR1vZvemVi82s1PN7IvxV23kSCS7qCjtO1afPms8T23aR1eXBvpFZPTLpQXzA+BqoAPA3Z8mumlSgkSyM+sU5XSnz5rA/kRSd/SLSFHIJcDUuHvPu+j1eMY0iY6uXmeQpZw+azyAuslEpCjkEmB2mdlxhMUjzWwJsLX3Q4pLagymL3Prx1BXXc4Tm/YOQq1ERIZWLnfyXwHcBJxoZluADcAlsdZqhIkCTN9dZCUlxutnjufxlxVgRGT0y6UF4+7+NqABONHdz83xuKIRjcHk9pUsnDuRF7YfYPeBRMy1EhEZWrlcFX8O4O4H3X1/SLsrviqNPLl2kQGcc1w9AI+sz/RQThGR0SNrF5mZnQicDNSZ2QfSdtUCVXFXbCRJJLsYX12eU97XTa9jTEUpD6/fxbtOnRpzzUREhk5vYzAnAO8GxgPvSUvfD3w8xjqNOImOTirGVeaUt7y0hIVzJ/Lnl3bHXCsRkaGVNcC4+93A3WZ2jrs/PIh1GnHa+9FFBlE32f3rdrK9tY0ptWoMisjolMsssifM7Aqi7rLuq6G7fzS2Wo0wuc4iSznn2EkAPPzSbt53+vS4qiUiMqRy+bP7x8AxwIXAH4AZRN1kEvRnFhlEC1/Wj6nggXU7YqyViMjQyuWqOM/dvwQcdPfbgHcBfxVvtUaW/swig+gJl285oYEHXthJp9YlE5FRKperYkf4uc/MTgHqgMnxVWnk6W8XGcAFJ05h36EOnnhFN12KyOiUS4C5ycwmAF8EVgBrgetirdUI4u79HuQHeNPxkygrMe59Xt1kIjI69XlVdPcfuvted/+jux/r7pOB3+ZSuJktMrN1ZtZsZldl2F9pZneE/Y+a2Zy0fVeH9HVmdmFfZZrZBWb2FzN70sz+ZGbzcqnjQHU/zbIfYzAAtVXlnDVnIvc9pwAjIqNTr1dFMzvHzJaY2eTw/lQz+wnwUF8Fm1kpcANwEbAAWGZmC3pkuwzY6+7zgOsJLaOQbynRzLVFwHfNrLSPMr8HXOLurwd+QtTiil0uj0vO5oKTJrNu+35e3n2w0NUSERlyWQOMmX0duAX4IPAbM/sP4PfAo8D8HMpeCDS7+3p3bweWA4t75FkM3Ba27wIusOixkIuB5e6ecPcNQHMor7cynWiVAYjGiV7NoY4Dlkh2AlDRzy4ygEWnHAPAr5/W4tQiMvr0dh/Mu4DT3b0tjMFsAk5x9405lj09HJOymdfOPuvO4+5JM2sB6kP6Iz2OTd0wkq3MjwErzeww0AqcnalSZnY5cDnArFmzcvwo2SU6Ui2Y/geYGRNqOH3WeH799FaueOug9OiJiAya3q6Kbe7eBuDue4EX+xFchsJngXe6+wzgv4BvZsrk7je5e6O7NzY0NAz4pEe6yPJbYPrdp07jua2tesqliIw6vV0VjzWzFakXMLfH+75sAWamvZ8R0jLmMbMyoq6t3b0cmzHdzBqA09z90ZB+B/CGHOo4YKkusnzGYADe9bqpmMFv1E0mIqNMb11kPcdL/k8/y34MmG9mc4kCw1Lgb3rkWQFcCjwMLAHuc3cPAewnZvZNYBrRmM9qwLKUuZdo1efj3f0F4O3Ac/2sb17a85xFlnJMXRVnzZ7I3U9u4R/Pn0c0BCUiMvL1ttjlHwZScBhTuRJYBZQCt7j7GjO7Bmhy9xXAzcCPzawZ2EMUMAj57iS65yYJXOHunQCZygzpHwd+bmZdRAFnUNZKG2gXGcAHz5zOv/z8Gf7yyl7OnD2xUFUTERlSuSx2mTd3Xwms7JH25bTtNuDiLMdeC1ybS5kh/ZfALwdY5X4byDTllHefOo1rfrWWOx7bpAAjIqOGHn08QImO1BhM/l/lmMoy3nPaNH711Fb2t3X0fYCIyAigADNAhegiA/jrs2ZyuKNT98SIyKjRZxeZmf2K6CbGdC1AE/D91FTmYlWILjKA02eO54Qp4/jRwy+z9KyZGuwXkREvlz+71wMHgB+EVyvR82COD++LWvc05TxnkaWYGR954xye29rKw+v1OGURGflyuSq+wd3/xt1/FV5/C5zl7lcAZ8Rcv2FvIHfy9/S+06dTP6aCW/60YcBliYgMtVyuimPNrHtNlbA9Nrxtj6VWI0h7Z2G6yACqyku55OzZ3Pv8Dtbrzn4RGeFyCTD/BPzJzO43sweAB4F/NrMxHFmosmilWjD5LHaZyd+dPZvykhJ+qFaMiIxwfQ7yu/tKM5sPnBiS1qUN7P/fuCo2UiSSnZSXGqUlhRmUbxhXycWNM7izaROfPO84ZkyoKUi5IiKDLdc/u88kejbLacBfm9nfx1elkSWfxyX35Yq3zsMwbrj/pYKWKyIymPoMMGb2Y+AbwLnAWeHVGHO9RoxEsrMgA/zppo2v5kNnzeRnTZvYtOdQQcsWERksuSwV0wgscPee98II0RhMocZf0n3yrcdxx2Ob+Pa9L/L1i08rePkiInHL5cr4LHBM3BUZqaIussIHmKl11fzdObO56y+bWfNqS8HLFxGJWy5XxknAWjNb1c/nwRSFqIussGMwKZ86fz7jq8u55ldrUQNSREaaXLrIvhJ3JUayRLJrwHfxZ1NXU87n3n48X7p7DavWbGfRKWpIisjIkcs05QE9F2a0a4+piyxl2cJZ/Ojhl7l25VrecnwD1RXxtJZERAot65XRzP4Ufu43s9a0134zax28Kg5vcUxTTldWWsK/v+8UNu05zPX//UJs5xERKbSsAcbdzw0/x7l7bdprnLvXDl4Vh7c4pin3dPax9SxbOIsfPriepzfvi/VcIiKFktOV0cxKzWyamc1KveKu2EiR6IhvDCbdVRedyKSxlXz+rqdpD48IEBEZznK50fIfge3APcBvwuvXMddrxEgku6gojT/A1FWX8x/vO4Xnt+3nm/eoq0xEhr9croyfBk5w95Pd/XXhdWouhZvZIjNbZ2bNZnZVhv2VZnZH2P+omc1J23d1SF9nZhf2VaZFrjWzF8zsOTP7VC51HKg4pyn39I6Tj2HZwpl8/48v8VDzrkE5p4hIvnIJMJuInmDZL2ZWCtwAXAQsAJaZ2YIe2S4D9rr7POB64Lpw7AJgKdH6Z4uA74Zuut7K/DAwEzjR3U8Clve3zvmIc5pyJl969wKOnTSGz97xJHsOFv3TEkRkGMv1iZYPhBbF51KvHI5bCDS7+3p3bye64C/ukWcxR5b8vwu4wKJnBS8Glrt7wt03AM2hvN7K/AfgGnfvAnD3HTnUccASHfFOU+6ppqKM7yw7g32HOvj08ifo7NINmCIyPOVyZXyFaPylAhiX9urLdKLWT8rmkJYxj7sniVpK9b0c21uZxwEfMrMmM/tteMTAa5jZ5SFP086dO3P4GL1r74x3mnImC6bV8r8Wn8yDL+7ia797flDPLSKSq15vtAxdUse7+yWDVJ+BqATa3L3RzD4A3AK8qWcmd78JuAmgsbFxQH/+Jzu76OzyQW3BpCxbOIu1r7by/T+u56Sptbzv9J6xW0RkaPV6ZXT3TmC2mVXkUfYWojGRlBkhLWMeMysD6oDdvRzbW5mbgV+E7V8COU1EGIhEmC48mGMw6b78ngUsnDuRf/n50zRt3DMkdRARySbXMZiHzOxL/RyDeQyYb2ZzQ4BaCvRcJHMFcGnYXgLcFx4LsAJYGmaZzQXmA6v7KPP/AW8N228BYp/L2x1gBrmLLKW8tITvXXIG08ZXc9ltTbywff+Q1ENEJJNcAsxLRPe9lNCPMZgwpnIlsAp4DrjT3deY2TVm9t6Q7Wag3syagc8BV4Vj1wB3AmuB3wFXuHtntjJDWV8FPmhmzwD/H/CxHD7bgCSSnQBD0kWWUj+2kh99dCEVZSVcestqtrYcHrK6iIiks2JeBr6xsdGbmpryPn7jroOc940H+OZfn8YHzphRwJr135pXW/jQ9x9h8rhKfnr52UyprRrS+ojI6GVmj7t7n082zuVO/gYz+7qZrTSz+1KvwlRzZBvqLrJ0J0+r49aPnMX21jaW3fQI21vbhrpKIlLkcunbuR14HpgL/C9gI9FYSNEbDl1k6RrnTOS2jy5ke2sbS296hG0tCjIiMnRyuTLWu/vNQIe7/8HdPwqcH3O9RoShnkWWSeOcifzosoXs3J/gg9/7M807NPAvIkMjlytjR/i51czeZWanAxNjrNOI0T6MusjSnTl7Ij/9+Nkkkp188HsPawqziAyJXALMf5hZHfBPwD8DPwQ+G2utRojh1kWW7nUz6vjFP7yRiWMquOSHj7Lyma1DXSURKTJ9Xhnd/dfu3uLuz7r7W939THfveT9LUUp0DL8usnSz6mu46xPnsGBaLZ+8/S98fdXzWrtMRAZNLrPIjjeze83s2fD+VDP7YvxVG/6G0yyybOrHVrL88rP5UONMbrj/JS677TFaDnf0faCIyADl8qf3D4CrCWMx7v400R30RS/VRVYxDLvI0lWWlfLVD76Oa99/Cg817+I93/kTT7yyd6irJSKjXC5Xxhp3X90jLRlHZUaaIy2Y4R1gAMyMS/5qNssvP4fOLufiGx/mhvub1WUmIrHJ5cq4y8yOAxzAzJYAGjEmbQxmBASYlDNnT2Dlp9/EolOO4eur1vE3P3iEzXsPDXW1RGQUyuXKeAXwfeBEM9sCfAb4RJyVGimOzCIbvmMwmdRVl/OdZafzjYtP49ktLbzj+j9y60Mb1JoRkYLKZRbZend/G9BA9Djic4H3x16zEaA92YUZlJfaUFel38yMJWfOYNVn38xZcybylV+t5eIb/8yLWpFZRAok574ddz/o7qmrTy7L9Y96iWT0uOToKc8j04wJNdz6kbO4/kOnsWHXQd757Qf53yufo7VNM81EZGDyHTwYuVfUAooCzMjqHsvEzHj/6TO453Nv4f2nT+cHD67n/G88wJ2PbaJL3WYikqd8A4yuOkRjMCNpgL8vk8ZW8rUlp3H3FW9kdv0YPv/zp1l8w0M8+OJOivmxDiKSn6xXRzPbb2atGV77gWmDWMdhK9HRNWzv4h+IU2eM565PnMO3lr6ePQfb+bubV7P0pke0ppmI9EtZth3u3udTK4tdItlFRenoCzAQdZstfv10Fp1yDMtXb+I79zWz5MaHOe+EBj51wXzOmDVhqKsoIsPc6Lw6DpKoi2zkj8H0prKslEvfMIcHP/9WrrroRJ7ctI8PfPfP/PX3H+b+53eo60xEslKAGYBEcnR2kWVSXVHKJ95yHH/6l/P54rtOYtOeQ3zk1se46FsP8ssnNtPR2TXUVRSRYSbWq6OZLTKzdWbWbGZXZdhfaWZ3hP2PmtmctH1Xh/R1ZnZhP8r8tpkdiO1DpUlNUy4mYyvL+NibjuUP//OtfOPi0+jscj57x1O88av3cf09L+hRzSLSLbaro5mVAjcAFwELgGVmtqBHtsuAve4+D7geuC4cu4BoQc2TgUXAd82stK8yzawRGLTBgdEyTTkfFWUl0Y2an3kzt3y4kZOm1vKte1/kDV+9j0/e/jgPv7Rb3WciRS7rIH8BLASa3X09gJktBxYDa9PyLAa+ErbvAv7TorsWFwPL3T0BbDCz5lAe2coMwefrwN8wSCsNJDo6qRxXORinGrZKSozzT5zC+SdO4eXdB7n90Ve4s2kTK5/ZxrGTxvDBM2fw/tOnM2189VBXVUQGWZz9O9OBTWnvN4e0jHncPQm0APW9HNtbmVcCK9y914U4zexyM2sys6adO3f26wP11J7sorK8OFswmcyuH8O/vvMkHrn6Ar5x8WlMGlfJ11et443X3cclP3yEnz++mUPtWohbpFjE2YIZNGY2DbgYOK+vvO5+E3ATQGNj44D6cIpxDCYXVeWlLDlzBkvOnMEruw/xiyc284u/bOGffvYUX7r7Wd520hTe+bqpnHdCA1UK0CKjVpwBZgswM+39jJCWKc9mMysD6oDdfRybKf10YB7QHNYFqzGz5jC2E5tEsnPYP2xsqM2qr+EzbzueT18wn8c27uWXT2zmd89uY8VTr1JTUcr5J07mXa+bynknTKa6QsFGZDSJM8A8Bsw3s7lEQWAp0fhIuhXApcDDwBLgPnd3M1sB/MTMvkm0asB8YDXRGmivKdPd1wDHpAo1swNxBxcId/IrwOTEzFg4dyIL507k3xefwiPr97Dy2a2senYbv356K9XlpZx3QgPnnziZ806YTEORj22JjAaxBRh3T5rZlcAqoBS4xd3XmNk1QJO7rwBuBn4cBvH3EB7FHPLdSTQhIAlc4e6dAJnKjOsz9KWYZ5ENRFlpCefOn8S58ydxzXtPZvXGPax8Ziv3rN3Ob5/dhlm0XM0FJ07m/BMnc/K02hG9YrVIsbJinkra2NjoTU1NeR3b1eUc+68r+fQF8/ns248vcM2Kk7uzdmsr9z23g3uf38FTm/fhDpPHVXLuvEm8Yd4k3jivnql1mpEmMpTM7HF3b+wr36gY5B8K7eHO9WK5k38wmBknT6vj5Gl1/OMF89l1IMED63Zy/7odPPDCTn7xRDQMd2zDmCjgHDeJc46tp66mfIhrLiKZKMDkKZEMAUZdZLGZNLayezZaV5fz/Lb9PNS8i4de2sXPmjbzo4dfpsRgwbRazpozkbPmTKRx9gQm11YNddVFBAWYvCWSnQAa5B8kJSXGgmm1LJhWy8fffCztyS6e3LSPPzXvYvWG3fx09Sv810MbAZhdX0Pj7ImcNWcCjXMmclzDGI3hiAwBBZg8JTpSLRgFmKFQUVbSPSsNopte17zaQtPGvTS9vIcH1u3g53/ZDEBddTmnzqjj1Bl1nDZjPKfNHM8UtXJEYqcAk6dUF5nugxkeKspKOH3WBE6fNYGPcyzuzoZdB3ls4x6e3NTCU5v2ceMf1tMZHgF9TG1VFHBmjufUGdG4z8QxFUP8KURGFwWYPB3pItMYzHBkZhzbMJZjG8byobOitMPtnazd2sJTm1p4avM+nt7cwu/Xbu8+ZkptJSdNreWkqbUsCD/nThpDaYm610TyoQCTp+5Bfs0iGzGqK0o5c/ZEzpw9sTut5VAHz2xp4bmtrTy3tZW1W1v504u7SIaWTlV5CSdMGRcFnWm1nHhMLfMmj1VrRyQHCjB50hjM6FBXU95902dKItlJ844DPLd1f3fgWbVmG8sfO7LOav2YCo6bPJb5k8cyb/JY5k8ex7zJY5lSW6kJBSKBAkyeuu+DURfZqFNZVtp9P06Ku7OttY112/bTvOMAzTsO8OKOA/zqqVdpbTuyQvS4yjKOC0Fn7qQxzJ00hjn1Y5gzqYaaCv13k+Ki3/g8JTo0TbmYmBlT66qZWlfNeSdM7k53d3YeSHQHneYdB3hx+wH+8MJO7np881FlTKmtZE59CDoh8MydNIbZ9TVaVVpGJQWYPKXGYKo0BlPUzIzJ46qYPK6KNxw36ah9+9s6eHn3ITbuPsjGXQfZsCvavmftdnYfbE8rA6aMq2LGhGpmTqyJfk6o6X4/ta6KslL9nsnIowCTJ93JL30ZV1XOKdPrOGV63Wv2tbZ1sHHXQTbuPsTGXQd5Zc8hNu89xOoNe7j7ycN0pS0RWFpiHFNbxcyJ1cyYUPOa4DOltkrT5WVYUoDJk+7kl4GorSrn1BnjOXXG+Nfs6+jsYltLG5v2HGLz3sNs2ht+7jnEgy/uZHtr4qj8ZtGyOtPqqjimrip05UXb08ZXc0xttF2uVpAMMgWYPKVmkekvRym08tISZk6sYebEmoz7E8lOtuw9zOa9h9nacpitLW1s3dfG1tY21u88yJ+bd7M/cfSjqTMFoYZxlTSMq2TyuMqom6+2kok1FZTovh8pEAWYPKmLTIZKZVlp902k2exv62BbSxuvtrSxreUwr+5rC+8PZw1CEHXHTRpbEcaVKplcW0nD2EoaasP7EJQaxlXqd1/6pACTp1QXmVowMhyNqypnXFU586eMy5rncHsnO/cn2LG/jR37E0e2WxPs2J/g1ZY2ntrcwu6DCTI9Nmp8TTmTx1VSP6aS+rEV1I+poH5sJRPHHL09aWwFtVXlahkVIQWYPCWSXZSXmpYRkRGruqKUWfU1zKrP3BWXkuzsYvfBdna0Jth54EgASgWj3QfbWfNqK7sOJNjf9tpWEUQtowk1UbCZGIJP/ZjU9tEBaUJNBbVVZZo5NwoowOSpXY9LliJRVlrClNqqsAL1a2fEpWtPdrH3UDu7DiTYc7Cd3Qfa2X2wnT0HE93buw8keGbzPnYfbM8akABqq8oYX1PBhJpyxtdUML6mnAnh5/jqciaMqYjSq0P6mHLGVZZpJYVhRAEmT4lkp2aQifRQUZYejPqWSHay92AHu0MA2nOwnX2H2tl7qIN9h9rZd7iDvYc62HuonQ27DrL3UO9BqbTEGF9dHgWhEHxqq8upqy6ntqqM2vC+tiqkVZdF2zXljK0oUzdegcUaYMxsEfAtoBT4obt/tcf+SuBHwJnAbuBD7r4x7LsauAzoBD7l7qt6K9PMbgcagQ5gNfA/3L0jrs+W6OhSgBEZoMqyUo6pK+WYutyfz5Ps7KIlBJ6Ww+3sPRgFoCitnX2HOtgXgtLWljZe2LGflkMd7E8kM44lpZhFS/3U1UQB6DVBKBWcqssYV1nO2KoyxlUd2R5bWaYx2R5iCzBmVgrcALwd2Aw8ZmYr3H1tWrbLgL3uPs/MlgLXAR8yswXAUuBkYBrw32Z2fDgmW5m3A38b8vwE+Bjwvbg+XyLZRaWW9xAZdGWlJdEYztjKfh3X1eUcaE/ScqiD1rYOWg8naTmc2g6vtiSthzu60zfsOti9fai9s89zVJaVREGnqpyxlVHQORKIUtvRvnEhfWzl0e/HVJaNmnuW4mzBLASa3X09gJktBxYD6QFmMfCVsH0X8J8WdaAuBpa7ewLYYGbNoTyylenuK1OFmtlqYEZcHwyipn3FKPklECkGJSXW3TLJR0dnV3cQOtCWZH9b1CpKbR9IJNmfSLI/7D+QiNI37TkUtqO0zq5emlFBRWkJYypLqamIglRNZSljK8sYU3FkO9p3JM+YyvR96XnKqCovGZKxqTgDzHRgU9r7zcBfZcvj7kkzawHqQ/ojPY6dHrZ7LdPMyoG/Az49wPr3KmrBKMCIFIvyPFtO6dydto6uHsEpyYFEB/vD9qH2JAcSneFnkkOJTg6G7R2tiSitPcnBRGf3qu59KTEYU3F0EPq39yw46tlIcRiNg/zfBf7o7g9m2mlmlwOXA8yaNSvvk2gMRkT6y8yoriiluqKUyX1n71N7sutIIGrv7A5IR4LQa4PVgfYkhxLJQZkFG2eA2QLMTHs/I6RlyrPZzMqI5kDu7uPYrGWa2b8BDcD/yFYpd78JuAmgsbGx77ZqFolkp57vISJDqqKshIqyaLr2cBTnn+CPAfPNbK6ZVRAN2q/okWcFcGnYXgLc5+4e0peaWaWZzQXmE80My1qmmX0MuBBY5u65tRsHoL1TLRgRkd7E9id4GFO5ElhFNKX4FndfY2bXAE3uvgK4GfhxGMTfQxQwCPnuJJoQkASucPdOgExlhlPeCLwMPBwGs37h7tfE9fkSHRqDERHpTax9PGFm18oeaV9O224DLs5y7LXAtbmUGdIHtb8qoTv5RUR6pT/B86Q7+UVEeqcrZJ6iFoy+PhGRbHSFzFOio0vLQoiI9EJXyDy4e+gi0xiMiEg2CjB5SHY5XY66yEREeqErZB66H5esacoiIlnpCpmH9lSAUReZiEhWCjB5SCSjZbvVRSYikp2ukHlIdKiLTESkL7pC5iGhLjIRkT4pwOQh1UWmB46JiGSnK2QeNItMRKRvukLmoXsMRl1kIiJZKcDkQbPIRET6pitkHtrVRSYi0iddIfOgWWQiIn1TgMmDushERPqmK2QejrRg9PWJiGSjK2QejtzJry4yEZFsFGDyoBstRUT6FusV0swWmdk6M2s2s6sy7K80szvC/kfNbE7avqtD+jozu7CvMs1sbiijOZRZEdfnSiS7MIPyUovrFCIiI15sAcbMSoEbgIuABcAyM1vQI9tlwF53nwdcD1wXjl0ALAVOBhYB3zWz0j7KvA64PpS1N5Qdi0Syi8qyEswUYEREsomzBbMQaHb39e7eDiwHFvfIsxi4LWzfBVxg0VV7MbDc3RPuvgFoDuVlLDMcc34og1Dm++L6YIkOPS5ZRKQvZTGWPR3YlPZ+M/BX2fK4e9LMWoD6kP5Ij2Onh+1MZdYD+9w9mSH/UczscuBygFmzZvXvEwUnTa3lcEdnXseKiBSLohuldveb3L3R3RsbGhryKmPpwll8bclpBa6ZiMjoEmeA2QLMTHs/I6RlzGNmZUAdsLuXY7Ol7wbGhzKynUtERAZRnAHmMWB+mN1VQTRov6JHnhXApWF7CXCfu3tIXxpmmc0F5gOrs5UZjrk/lEEo8+4YP5uIiPQhtjGYMKZyJbAKKAVucfc1ZnYN0OTuK4CbgR+bWTOwhyhgEPLdCawFksAV7t4JkKnMcMp/AZab2X8AT4SyRURkiFj0x39xamxs9KampqGuhojIiGJmj7t7Y1/5im6QX0REBocCjIiIxEIBRkREYqEAIyIisSjqQX4z2wm8nOfhk4BdBaxOoahe/aN69Y/q1T/DtV4wsLrNdvc+71Qv6gAzEGbWlMssisGmevWP6tU/qlf/DNd6weDUTV1kIiISCwUYERGJhQJM/m4a6gpkoXr1j+rVP6pX/wzXesEg1E1jMCIiEgu1YEREJBYKMCIiEg9316ufL2ARsI7oUc5XxVD+TKLHD6wF1gCfDulfIXrOzZPh9c60Y64O9VkHXNhXXYG5wKMh/Q6gIse6bQSeCedvCmkTgXuAF8PPCSHdgG+HczwNnJFWzqUh/4vApWnpZ4bym8OxlkOdTkj7Tp4EWoHPDNX3BdwC7ACeTUuL/TvKdo4+6vV14Plw7l8C40P6HOBw2nd3Y77n7+0z9lKv2P/tgMrwvjnsn5NDve5Iq9NG4MnB/L7Ifm0Y8t+vjP8XCn1xHO0voscEvAQcC1QATwELCnyOqalfBGAc8AKwIPyn++cM+ReEelSG/0wvhXpmrStwJ7A0bN8I/EOOddsITOqR9jXCf2jgKuC6sP1O4Lfhl/xs4NG0X9T14eeEsJ36D7E65LVw7EV5/PtsA2YP1fcFvBk4g6MvTLF/R9nO0Ue93gGUhe3r0uo1Jz1fj3L6df5sn7GPesX+bwd8khAIiB4Vckdf9eqx//8AXx7M74vs14Yh//3K+Nn7e/Er9hdwDrAq7f3VwNUxn/Nu4O29/Kc7qg5Ez8s5J1tdwy/OLo5cWI7K10ddNvLaALMOmBq2pwLrwvb3gWU98wHLgO+npX8/pE0Fnk9LPypfjvV7B/BQ2B6y74seF5zB+I6ynaO3evXY937g9t7y5XP+bJ+xj+8r9n+71LFhuyzks97qlZZuwCZg/lB8X2n7UteGYfH71fOlMZj+m070i5WyOaTFwszmAKcTNeEBrjSzp83sFjOb0EedsqXXA/vcPdkjPRcO/N7MHjezy0PaFHffGra3AVPyrNf0sN0zvT+WAj9Nez/U31fKYHxH2c6Rq48S/cWaMtfMnjCzP5jZm9Lq29/z5/t/Ju5/u+5jwv6WkD8XbwK2u/uLaWmD+n31uDYMy98vBZhhzMzGAj8HPuPurcD3gOOA1wNbiZrog+1cdz8DuAi4wszenL7Toz9vfAjqRXiM9nuBn4Wk4fB9vcZgfEf9PYeZfYHo6bG3h6StwCx3Px34HPATM6uN6/wZDMt/uzTLOPoPmUH9vjJcG/IuKx+5nkMBpv+2EA20pcwIaQVlZuVEv0C3u/svANx9u7t3unsX8ANgYR91ypa+GxhvZmU90vvk7lvCzx1Eg8ILge1mNjXUeyrRwGg+9doStnum5+oi4C/uvj3Ucci/rzSD8R1lO0evzOzDwLuBS8KFA3dPuPvusP040fjG8Xmev9//Zwbp3677mLC/LuTvVcj7AaIB/1R9B+37ynRtyKOsQfn9UoDpv8eA+WY2N/zFvBRYUcgTmJkBNwPPufs309KnpmV7P/Bs2F4BLDWzSjObC8wnGqjLWNdwEbkfWBKOv5SoL7eveo0xs3GpbaLxjmfD+S/NUNYK4O8tcjbQEprYq4B3mNmE0PXxDqJ+8a1Aq5mdHb6Dv8+lXmmO+qtyqL+vHgbjO8p2jqzMbBHweeC97n4oLb3BzErD9rFE39H6PM+f7TP2Vq/B+LdLr+8S4L5UgO3D24jGKbq7kgbr+8p2bcijrEH5/SroYHSxvIhmZrxA9FfKF2Io/1yi5ufTpE3TBH5MNH3w6fCPPTXtmC+E+qwjbeZVtroSzbZZTTQV8WdAZQ71OpZods5TRFMkvxDS64F7iaYv/jcwMaQbcEM49zNAY1pZHw3nbgY+kpbeSHQxeQn4T3KYphyOG0P012ddWtqQfF9EQW4r0EHUh33ZYHxH2c7RR72aifriU79nqVlVHwz/xk8CfwHek+/5e/uMvdQr9n87oCq8bw77j+2rXiH9VuATPfIOyvdF9mvDkP9+ZXppqRgREYmFushERCQWCjAiIhILBRgREYmFAoyIiMRCAUZERGKhACPST2ZWb2ZPhtc2M9uS9r6ij2Mbzezb/TzfR83sGYuWTXnWzBaH9A+b2bSBfBaROGmassgAmNlXgAPu/o20tDI/svbVQMufAfyBaAXdlrBESIO7bzCzB4gWhGwqxLlECk0tGJECMLNbzexGM3sU+JqZLTSzhy1a/PDPZnZCyHeemf06bH/FooUcHzCz9Wb2qQxFTwb2AwcA3P1ACC5LiG6Iuz20nKrN7EyLFlp83MxW2ZFlPR4ws2+FfM+a2cIM5xEpOAUYkcKZAbzB3T9H9BCvN3m0+OGXgf+d5ZgTgQuJ1tr6N4vWmUr3FLAd2GBm/2Vm7wFw97uAJqL1w15PtFDld4Al7n4m0cOyrk0rpybk+2TYJxK7sr6ziEiOfubunWG7DrjNzOYTLe3RM3Ck/MbdE0DCzHYQLYHevcaVu3eG9cLOAi4ArjezM939Kz3KOQE4BbgnWkKKUqJlTlJ+Gsr7o5nVmtl4d9+X/0cV6ZsCjEjhHEzb/nfgfnd/v0XP7XggyzGJtO1OMvyf9GigdDWw2szuAf6L6IFc6QxY4+7nZDlPz8FWDb5K7NRFJhKPOo4sc/7hfAsxs2lmdkZa0uuBl8P2fqLH5kK08GODmZ0Tjis3s5PTjvtQSD+XaEXdlnzrJJIrtWBE4vE1oi6yLwK/GUA55cA3wnTkNmAn8Imw71bgRjM7TPQo4CXAt82sjuj/9v8lWuEXoM3MngjlfXQA9RHJmaYpi4xyms4sQ0VdZCIiEgu1YEREJBZqwYiISCwUYEREJBYKMCIiEgsFGBERiYUCjIiIxOL/BxWPw2YhM9c1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample_learning_rate = CustomSchedule(d_model=128)\n",
    "\n",
    "plt.plot(sample_learning_rate(tf.range(200000, dtype=tf.float32)))\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.xlabel(\"Train Step\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cedaf81",
   "metadata": {},
   "source": [
    "### 4. 모델 컴파일"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "aeaef27f",
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(D_MODEL)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(\n",
    "    learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)\n",
    "\n",
    "def accuracy(y_true, y_pred):\n",
    "  y_true = tf.reshape(y_true, shape=(-1, MAX_LENGTH - 1))\n",
    "  return tf.keras.metrics.sparse_categorical_accuracy(y_true, y_pred)\n",
    "\n",
    "model.compile(optimizer=optimizer, loss=loss_function, metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3570862a",
   "metadata": {},
   "source": [
    "### 5. 훈련하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "60c3c1af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.4269 - accuracy: 0.1030\n",
      "Epoch 2/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.3471 - accuracy: 0.1142\n",
      "Epoch 3/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.2734 - accuracy: 0.1247\n",
      "Epoch 4/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.2075 - accuracy: 0.1351\n",
      "Epoch 5/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.1523 - accuracy: 0.1446\n",
      "Epoch 6/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.1098 - accuracy: 0.1522\n",
      "Epoch 7/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0792 - accuracy: 0.1582\n",
      "Epoch 8/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0618 - accuracy: 0.1610\n",
      "Epoch 9/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0507 - accuracy: 0.1628\n",
      "Epoch 10/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0446 - accuracy: 0.1640\n",
      "Epoch 11/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0427 - accuracy: 0.1640\n",
      "Epoch 12/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0402 - accuracy: 0.1643\n",
      "Epoch 13/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0350 - accuracy: 0.1655\n",
      "Epoch 14/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0314 - accuracy: 0.1664\n",
      "Epoch 15/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0277 - accuracy: 0.1674\n",
      "Epoch 16/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0246 - accuracy: 0.1682\n",
      "Epoch 17/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0222 - accuracy: 0.1687\n",
      "Epoch 18/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0201 - accuracy: 0.1692\n",
      "Epoch 19/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.0187 - accuracy: 0.1696\n",
      "Epoch 20/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0166 - accuracy: 0.1700\n",
      "Epoch 21/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0154 - accuracy: 0.1703\n",
      "Epoch 22/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0143 - accuracy: 0.1708\n",
      "Epoch 23/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0133 - accuracy: 0.1710\n",
      "Epoch 24/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0120 - accuracy: 0.1712\n",
      "Epoch 25/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0114 - accuracy: 0.1715\n",
      "Epoch 26/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0115 - accuracy: 0.1713\n",
      "Epoch 27/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0097 - accuracy: 0.1718\n",
      "Epoch 28/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0099 - accuracy: 0.1719\n",
      "Epoch 29/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0089 - accuracy: 0.1721\n",
      "Epoch 30/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0088 - accuracy: 0.1722\n",
      "Epoch 31/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0080 - accuracy: 0.1723\n",
      "Epoch 32/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0076 - accuracy: 0.1723\n",
      "Epoch 33/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0076 - accuracy: 0.1724\n",
      "Epoch 34/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.0070 - accuracy: 0.1724\n",
      "Epoch 35/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0066 - accuracy: 0.1726\n",
      "Epoch 36/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0068 - accuracy: 0.1726\n",
      "Epoch 37/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0063 - accuracy: 0.1727\n",
      "Epoch 38/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.0059 - accuracy: 0.1727\n",
      "Epoch 39/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.0058 - accuracy: 0.1728\n",
      "Epoch 40/50\n",
      "185/185 [==============================] - 10s 54ms/step - loss: 0.0058 - accuracy: 0.1728\n",
      "Epoch 41/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0052 - accuracy: 0.1729\n",
      "Epoch 42/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0055 - accuracy: 0.1729\n",
      "Epoch 43/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0052 - accuracy: 0.1730\n",
      "Epoch 44/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0049 - accuracy: 0.1730\n",
      "Epoch 45/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0048 - accuracy: 0.1730\n",
      "Epoch 46/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0046 - accuracy: 0.1731\n",
      "Epoch 47/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0044 - accuracy: 0.1731\n",
      "Epoch 48/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0043 - accuracy: 0.1731\n",
      "Epoch 49/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0044 - accuracy: 0.1731\n",
      "Epoch 50/50\n",
      "185/185 [==============================] - 10s 53ms/step - loss: 0.0040 - accuracy: 0.1732\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7970bced92b0>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EPOCHS = 50\n",
    "model.fit(dataset, epochs=EPOCHS, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7efde1a8",
   "metadata": {},
   "source": [
    "## Step 5. 모델 평가하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2f75fa2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoder_inference(sentence):\n",
    "  sentence = preprocess_sentence(sentence)\n",
    "\n",
    "  # 입력된 문장을 정수 인코딩 후, 시작 토큰과 종료 토큰을 앞뒤로 추가.\n",
    "  # ex) Where have you been? → [[8331   86   30    5 1059    7 8332]]\n",
    "  sentence = tf.expand_dims(\n",
    "      START_TOKEN + tokenizer.encode(sentence) + END_TOKEN, axis=0)\n",
    "\n",
    "  # 디코더의 현재까지의 예측한 출력 시퀀스가 지속적으로 저장되는 변수.\n",
    "  # 처음에는 예측한 내용이 없음으로 시작 토큰만 별도 저장. ex) 8331\n",
    "  output_sequence = tf.expand_dims(START_TOKEN, 0)\n",
    "\n",
    "  # 디코더의 인퍼런스 단계\n",
    "  for i in range(MAX_LENGTH):\n",
    "    # 디코더는 최대 MAX_LENGTH의 길이만큼 다음 단어 예측을 반복합니다.\n",
    "    predictions = model(inputs=[sentence, output_sequence], training=False)\n",
    "    predictions = predictions[:, -1:, :]\n",
    "\n",
    "    # 현재 예측한 단어의 정수\n",
    "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "    # 만약 현재 예측한 단어가 종료 토큰이라면 for문을 종료\n",
    "    if tf.equal(predicted_id, END_TOKEN[0]):\n",
    "      break\n",
    "\n",
    "    # 예측한 단어들은 지속적으로 output_sequence에 추가됩니다.\n",
    "    # 이 output_sequence는 다시 디코더의 입력이 됩니다.\n",
    "    output_sequence = tf.concat([output_sequence, predicted_id], axis=-1)\n",
    "\n",
    "  return tf.squeeze(output_sequence, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a09b399",
   "metadata": {},
   "source": [
    "### 챗봇의 대답을 얻는 sentence_generation() 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "feedfa1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_generation(sentence):\n",
    "  # 입력 문장에 대해서 디코더를 동작 시켜 예측된 정수 시퀀스를 리턴받습니다.\n",
    "  prediction = decoder_inference(sentence)\n",
    "\n",
    "  # 정수 시퀀스를 다시 텍스트 시퀀스로 변환합니다.\n",
    "  predicted_sentence = tokenizer.decode(\n",
    "      [i for i in prediction if i < tokenizer.vocab_size])\n",
    "\n",
    "  print('입력 : {}'.format(sentence))\n",
    "  print('출력 : {}'.format(predicted_sentence))\n",
    "\n",
    "  return predicted_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "54c0d2aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 오늘 오전에 눈이 왔지?\n",
      "출력 : 그게 제일 힘들어요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'그게 제일 힘들어요 .'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation('오늘 오전에 눈이 왔지?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a07e626f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 과제가 쉽지 않지\n",
      "출력 : 시간이 필요한 것 같아요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'시간이 필요한 것 같아요 .'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation(\"과제가 쉽지 않지\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0e15876c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 너는 누구야?\n",
      "출력 : 마음이 조급해지면 하지 말고 쉬세요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'마음이 조급해지면 하지 말고 쉬세요 .'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation(\"너는 누구야?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1ec31402",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 너의 역할은?\n",
      "출력 : 서로 다르게 태어난 이유는 저마다의 목소리를 내기 위해서예요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'서로 다르게 태어난 이유는 저마다의 목소리를 내기 위해서예요 .'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation(\"너의 역할은?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "46d58035",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 뭐야?\n",
      "출력 : 직접 물어보는 게 좋을 것 같아요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'직접 물어보는 게 좋을 것 같아요 .'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation(\"뭐야?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9a8c3d5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : 위로를 할줄 알아?\n",
      "출력 : 저도 좋아해요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'저도 좋아해요 .'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation(\"위로를 할줄 알아?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e2ff43a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "입력 : ㅎㅎ\n",
      "출력 : 좋은 곳으로 가보세요 .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'좋은 곳으로 가보세요 .'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_generation(\"ㅎㅎ\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "397a2754",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}