chapter11_part04_sequence-to-sequence-learning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wnO2lSxrC6Yd"
      },
      "source": [
        "이 노트북은 [케라스 창시자에게 배우는 딥러닝 2판](https://tensorflow.blog/kerasdl2/)의 예제 코드를 담고 있습니다.\n",
        "\n",
        "<table align=\"left\">\n",
        "    <tr>\n",
        "        <td>\n",
        "            <a href=\"https://colab.research.google.com/github/rickiepark/deep-learning-with-python-2nd/blob/main/chapter11_part04_sequence-to-sequence-learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
        "        </td>\n",
        "    </tr>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hYUxgYWrC6Yg"
      },
      "source": [
        "## 텍스트 분류를 넘어: 시퀀스-투-시퀀스 학습"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GFpTmSrxC6Yg"
      },
      "source": [
        "### 기계 번역 예제"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I1ZDSF7AC6Yh",
        "outputId": "5bfb134c-ff24-463d-e5ad-5d8fc1757164"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-11-22 07:34:14--  http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 74.125.201.128, 74.125.202.128, 74.125.69.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|74.125.201.128|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2638744 (2.5M) [application/zip]\n",
            "Saving to: ‘spa-eng.zip’\n",
            "\n",
            "\rspa-eng.zip           0%[                    ]       0  --.-KB/s               \rspa-eng.zip         100%[===================>]   2.52M  --.-KB/s    in 0.01s   \n",
            "\n",
            "2022-11-22 07:34:14 (203 MB/s) - ‘spa-eng.zip’ saved [2638744/2638744]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip\n",
        "!unzip -q spa-eng.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "HwGu6EZxC6Yh"
      },
      "outputs": [],
      "source": [
        "text_file = \"spa-eng/spa.txt\"\n",
        "with open(text_file) as f:\n",
        "    lines = f.read().split(\"\\n\")[:-1]\n",
        "text_pairs = []\n",
        "for line in lines:\n",
        "    english, spanish = line.split(\"\\t\")\n",
        "    spanish = \"[start] \" + spanish + \" [end]\"\n",
        "    text_pairs.append((english, spanish))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j3ARnP9KC6Yi",
        "outputId": "29329afc-5a36-4a42-d8d5-c89b4f699846"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "('The game was canceled due to rain.', '[start] El partido se suspendió a causa de la lluvia. [end]')\n"
          ]
        }
      ],
      "source": [
        "import random\n",
        "print(random.choice(text_pairs))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "gcGm7oMfC6Yi"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "random.shuffle(text_pairs)\n",
        "num_val_samples = int(0.15 * len(text_pairs))\n",
        "num_train_samples = len(text_pairs) - 2 * num_val_samples\n",
        "train_pairs = text_pairs[:num_train_samples]\n",
        "val_pairs = text_pairs[num_train_samples:num_train_samples + num_val_samples]\n",
        "test_pairs = text_pairs[num_train_samples + num_val_samples:]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1c9yTMG4C6Yj"
      },
      "source": [
        "**영어와 스페인어 텍스트 쌍을 벡터화하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "vQd-CYfvC6Yj"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers\n",
        "import string\n",
        "import re\n",
        "\n",
        "strip_chars = string.punctuation + \"¿\"\n",
        "strip_chars = strip_chars.replace(\"[\", \"\")\n",
        "strip_chars = strip_chars.replace(\"]\", \"\")\n",
        "\n",
        "def custom_standardization(input_string):\n",
        "    lowercase = tf.strings.lower(input_string)\n",
        "    return tf.strings.regex_replace(\n",
        "        lowercase, f\"[{re.escape(strip_chars)}]\", \"\")\n",
        "\n",
        "vocab_size = 15000\n",
        "sequence_length = 20\n",
        "\n",
        "source_vectorization = layers.TextVectorization(\n",
        "    max_tokens=vocab_size,\n",
        "    output_mode=\"int\",\n",
        "    output_sequence_length=sequence_length,\n",
        ")\n",
        "target_vectorization = layers.TextVectorization(\n",
        "    max_tokens=vocab_size,\n",
        "    output_mode=\"int\",\n",
        "    output_sequence_length=sequence_length + 1,\n",
        "    standardize=custom_standardization,\n",
        ")\n",
        "train_english_texts = [pair[0] for pair in train_pairs]\n",
        "train_spanish_texts = [pair[1] for pair in train_pairs]\n",
        "source_vectorization.adapt(train_english_texts)\n",
        "target_vectorization.adapt(train_spanish_texts)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4tuHosZOC6Yk"
      },
      "source": [
        "**번역 작업을 위한 데이터셋 준비하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "M7e9MC6PC6Yk"
      },
      "outputs": [],
      "source": [
        "batch_size = 64\n",
        "\n",
        "def format_dataset(eng, spa):\n",
        "    eng = source_vectorization(eng)\n",
        "    spa = target_vectorization(spa)\n",
        "    return ({\n",
        "        \"english\": eng,\n",
        "        \"spanish\": spa[:, :-1],\n",
        "    }, spa[:, 1:])\n",
        "\n",
        "def make_dataset(pairs):\n",
        "    eng_texts, spa_texts = zip(*pairs)\n",
        "    eng_texts = list(eng_texts)\n",
        "    spa_texts = list(spa_texts)\n",
        "    dataset = tf.data.Dataset.from_tensor_slices((eng_texts, spa_texts))\n",
        "    dataset = dataset.batch(batch_size)\n",
        "    dataset = dataset.map(format_dataset, num_parallel_calls=4)\n",
        "    return dataset.shuffle(2048).prefetch(16).cache()\n",
        "\n",
        "train_ds = make_dataset(train_pairs)\n",
        "val_ds = make_dataset(val_pairs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VzIe7PCoC6Yl",
        "outputId": "fea09c37-04ba-424a-97d1-d5cd4c90ee12"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "inputs['english'].shape: (64, 20)\n",
            "inputs['spanish'].shape: (64, 20)\n",
            "targets.shape: (64, 20)\n"
          ]
        }
      ],
      "source": [
        "for inputs, targets in train_ds.take(1):\n",
        "    print(f\"inputs['english'].shape: {inputs['english'].shape}\")\n",
        "    print(f\"inputs['spanish'].shape: {inputs['spanish'].shape}\")\n",
        "    print(f\"targets.shape: {targets.shape}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D1tqIpNxC6Yl"
      },
      "source": [
        "### RNN을 사용한 시퀀스-투-시퀀스 모델"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Idn1gcR-C6Yl"
      },
      "source": [
        "**기반 인코더**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "RScgqDXWC6Ym"
      },
      "outputs": [],
      "source": [
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "embed_dim = 256\n",
        "latent_dim = 1024\n",
        "\n",
        "source = keras.Input(shape=(None,), dtype=\"int64\", name=\"english\")\n",
        "x = layers.Embedding(vocab_size, embed_dim, mask_zero=True)(source)\n",
        "encoded_source = layers.Bidirectional(\n",
        "    layers.GRU(latent_dim), merge_mode=\"sum\")(x)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xq5pLFNhC6Ym"
      },
      "source": [
        "**GRU 기반 디코더와 엔드-투-엔드 모델**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "-ZprypXUC6Ym"
      },
      "outputs": [],
      "source": [
        "past_target = keras.Input(shape=(None,), dtype=\"int64\", name=\"spanish\")\n",
        "x = layers.Embedding(vocab_size, embed_dim, mask_zero=True)(past_target)\n",
        "decoder_gru = layers.GRU(latent_dim, return_sequences=True)\n",
        "x = decoder_gru(x, initial_state=encoded_source)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "target_next_step = layers.Dense(vocab_size, activation=\"softmax\")(x)\n",
        "seq2seq_rnn = keras.Model([source, past_target], target_next_step)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D-kMw9qCC6Yn"
      },
      "source": [
        "**RNN 기반 시퀀스-투-시퀀스 모델 훈련하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n6Yp_XqJC6Yn",
        "outputId": "4a37b4fe-df74-4e6b-af5e-bb320e557f7f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/15\n",
            "1302/1302 [==============================] - 58s 31ms/step - loss: 1.6337 - accuracy: 0.4172 - val_loss: 1.3244 - val_accuracy: 0.5047\n",
            "Epoch 2/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 1.3152 - accuracy: 0.5253 - val_loss: 1.1645 - val_accuracy: 0.5664\n",
            "Epoch 3/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 1.1726 - accuracy: 0.5758 - val_loss: 1.0834 - val_accuracy: 0.5987\n",
            "Epoch 4/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 1.0822 - accuracy: 0.6084 - val_loss: 1.0560 - val_accuracy: 0.6138\n",
            "Epoch 5/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 1.0356 - accuracy: 0.6325 - val_loss: 1.0351 - val_accuracy: 0.6273\n",
            "Epoch 6/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 1.0044 - accuracy: 0.6513 - val_loss: 1.0327 - val_accuracy: 0.6314\n",
            "Epoch 7/15\n",
            "1302/1302 [==============================] - 37s 29ms/step - loss: 0.9849 - accuracy: 0.6644 - val_loss: 1.0328 - val_accuracy: 0.6354\n",
            "Epoch 8/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9710 - accuracy: 0.6753 - val_loss: 1.0339 - val_accuracy: 0.6386\n",
            "Epoch 9/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9614 - accuracy: 0.6831 - val_loss: 1.0369 - val_accuracy: 0.6388\n",
            "Epoch 10/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9536 - accuracy: 0.6890 - val_loss: 1.0388 - val_accuracy: 0.6403\n",
            "Epoch 11/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9481 - accuracy: 0.6932 - val_loss: 1.0410 - val_accuracy: 0.6410\n",
            "Epoch 12/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9440 - accuracy: 0.6963 - val_loss: 1.0440 - val_accuracy: 0.6424\n",
            "Epoch 13/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9418 - accuracy: 0.6984 - val_loss: 1.0453 - val_accuracy: 0.6413\n",
            "Epoch 14/15\n",
            "1302/1302 [==============================] - 37s 28ms/step - loss: 0.9398 - accuracy: 0.7004 - val_loss: 1.0463 - val_accuracy: 0.6424\n",
            "Epoch 15/15\n",
            "1302/1302 [==============================] - 37s 29ms/step - loss: 0.9399 - accuracy: 0.7008 - val_loss: 1.0467 - val_accuracy: 0.6426\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7faa64b24650>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "seq2seq_rnn.compile(\n",
        "    optimizer=\"rmsprop\",\n",
        "    loss=\"sparse_categorical_crossentropy\",\n",
        "    metrics=[\"accuracy\"])\n",
        "seq2seq_rnn.fit(train_ds, epochs=15, validation_data=val_ds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NUnTqtHQC6Yn"
      },
      "source": [
        "**RNN 인코더와 디코더로 새로운 문장 번역하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u1he-U_1C6Yo",
        "outputId": "fd32942f-51b9-4384-b28f-7739fcf0f8db"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-\n",
            "I was speechless with emotion.\n",
            "1/1 [==============================] - 3s 3s/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 38ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "[start] me estaba [UNK] con el sus de los de lo de los dijo [end]\n",
            "-\n",
            "Won't you join our conversation?\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] no [UNK] a nuestra nuestra [end]\n",
            "-\n",
            "Do it again!\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "[start] otra vez [end]\n",
            "-\n",
            "He could not forsake his friend in trouble.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] Él no podía a su amigo en su trabajo [end]\n",
            "-\n",
            "He likes to sing popular songs.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] le gusta cantar a los escribir los [UNK] [end]\n",
            "-\n",
            "That's a good idea.\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] esa es una buena idea [end]\n",
            "-\n",
            "I intend to phone Tom tomorrow and ask him to help.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 24ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "[start] yo puedo hacer a tom mañana con mi hermana y yo le [UNK] [end]\n",
            "-\n",
            "It really depends on when.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] realmente realmente en cuando [end]\n",
            "-\n",
            "Did you find Tom?\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "[start] le encontrar a tom [end]\n",
            "-\n",
            "Your friendship has great value to me.\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "[start] tu tu me ha sido mucho para mí [end]\n",
            "-\n",
            "He makes it a rule to take a walk every morning.\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "[start] tiene una buena para hacer lo que se me [UNK] por la mañana [end]\n",
            "-\n",
            "He has nothing to do with it.\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 23ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] Él no tiene nada que ver con eso [end]\n",
            "-\n",
            "It's very beautiful.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "[start] es muy inteligente [end]\n",
            "-\n",
            "Belgrade is the capital of Serbia.\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "[start] [UNK] es la de la [UNK] de lo más [end]\n",
            "-\n",
            "My father disapproved of my going to the concert.\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] mi padre se [UNK] a la ir al cine [end]\n",
            "-\n",
            "The room is very clean.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 36ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] la habitación está muy ocupado [end]\n",
            "-\n",
            "There's no doubt that Tom used to live in Boston.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "[start] no hay tom que [UNK] de vivir en boston [end]\n",
            "-\n",
            "So what do you want with me?\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 23ms/step\n",
            "1/1 [==============================] - 0s 19ms/step\n",
            "[start] qué quieres hacer conmigo [end]\n",
            "-\n",
            "I always keep promises.\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 22ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] siempre he [UNK] las en [end]\n",
            "-\n",
            "Where's your sister?\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 20ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "1/1 [==============================] - 0s 21ms/step\n",
            "[start] dónde está tu hermana [end]\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "spa_vocab = target_vectorization.get_vocabulary()\n",
        "spa_index_lookup = dict(zip(range(len(spa_vocab)), spa_vocab))\n",
        "max_decoded_sentence_length = 20\n",
        "\n",
        "def decode_sequence(input_sentence):\n",
        "    tokenized_input_sentence = source_vectorization([input_sentence])\n",
        "    decoded_sentence = \"[start]\"\n",
        "    for i in range(max_decoded_sentence_length):\n",
        "        tokenized_target_sentence = target_vectorization([decoded_sentence])\n",
        "        next_token_predictions = seq2seq_rnn.predict(\n",
        "            [tokenized_input_sentence, tokenized_target_sentence])\n",
        "        sampled_token_index = np.argmax(next_token_predictions[0, i, :])\n",
        "        sampled_token = spa_index_lookup[sampled_token_index]\n",
        "        decoded_sentence += \" \" + sampled_token\n",
        "        if sampled_token == \"[end]\":\n",
        "            break\n",
        "    return decoded_sentence\n",
        "\n",
        "test_eng_texts = [pair[0] for pair in test_pairs]\n",
        "for _ in range(20):\n",
        "    input_sentence = random.choice(test_eng_texts)\n",
        "    print(\"-\")\n",
        "    print(input_sentence)\n",
        "    print(decode_sequence(input_sentence))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ThpTHoiqC6Yo"
      },
      "source": [
        "### 트랜스포머를 사용한 시퀀스-투-시퀀스 모델"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pdRs7iLLC6Yo"
      },
      "source": [
        "#### 트랜스포머 디코더"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3bgMELmgC6Yo"
      },
      "source": [
        "**`TransformerDecoder` 클래스**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "NMlKqNtqC6Yo"
      },
      "outputs": [],
      "source": [
        "class TransformerDecoder(layers.Layer):\n",
        "    def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        self.embed_dim = embed_dim\n",
        "        self.dense_dim = dense_dim\n",
        "        self.num_heads = num_heads\n",
        "        self.attention_1 = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim)\n",
        "        self.attention_2 = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim)\n",
        "        self.dense_proj = keras.Sequential(\n",
        "            [layers.Dense(dense_dim, activation=\"relu\"),\n",
        "             layers.Dense(embed_dim),]\n",
        "        )\n",
        "        self.layernorm_1 = layers.LayerNormalization()\n",
        "        self.layernorm_2 = layers.LayerNormalization()\n",
        "        self.layernorm_3 = layers.LayerNormalization()\n",
        "        self.supports_masking = True\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super().get_config()\n",
        "        config.update({\n",
        "            \"embed_dim\": self.embed_dim,\n",
        "            \"num_heads\": self.num_heads,\n",
        "            \"dense_dim\": self.dense_dim,\n",
        "        })\n",
        "        return config\n",
        "\n",
        "    def get_causal_attention_mask(self, inputs):\n",
        "        input_shape = tf.shape(inputs)\n",
        "        batch_size, sequence_length = input_shape[0], input_shape[1]\n",
        "        i = tf.range(sequence_length)[:, tf.newaxis]\n",
        "        j = tf.range(sequence_length)\n",
        "        mask = tf.cast(i >= j, dtype=\"int32\")\n",
        "        mask = tf.reshape(mask, (1, input_shape[1], input_shape[1]))\n",
        "        mult = tf.concat(\n",
        "            [tf.expand_dims(batch_size, -1),\n",
        "             tf.constant([1, 1], dtype=tf.int32)], axis=0)\n",
        "        return tf.tile(mask, mult)\n",
        "\n",
        "    def call(self, inputs, encoder_outputs, mask=None):\n",
        "        causal_mask = self.get_causal_attention_mask(inputs)\n",
        "        if mask is not None:\n",
        "            padding_mask = tf.cast(\n",
        "                mask[:, tf.newaxis, :], dtype=\"int32\")\n",
        "            padding_mask = tf.minimum(padding_mask, causal_mask)\n",
        "        attention_output_1 = self.attention_1(\n",
        "            query=inputs,\n",
        "            value=inputs,\n",
        "            key=inputs,\n",
        "            attention_mask=causal_mask)\n",
        "        attention_output_1 = self.layernorm_1(inputs + attention_output_1)\n",
        "        attention_output_2 = self.attention_2(\n",
        "            query=attention_output_1,\n",
        "            value=encoder_outputs,\n",
        "            key=encoder_outputs,\n",
        "            attention_mask=padding_mask,\n",
        "        )\n",
        "        attention_output_2 = self.layernorm_2(\n",
        "            attention_output_1 + attention_output_2)\n",
        "        proj_output = self.dense_proj(attention_output_2)\n",
        "        return self.layernorm_3(attention_output_2 + proj_output)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yF6EcciHC6Yp"
      },
      "source": [
        "#### 기계 번역을 위한 트랜스포머"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_sOybVcwC6Yp"
      },
      "source": [
        "**`PositionalEmbedding` 층**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "L5e12udRC6Yp"
      },
      "outputs": [],
      "source": [
        "class PositionalEmbedding(layers.Layer):\n",
        "    def __init__(self, sequence_length, input_dim, output_dim, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        self.token_embeddings = layers.Embedding(\n",
        "            input_dim=input_dim, output_dim=output_dim)\n",
        "        self.position_embeddings = layers.Embedding(\n",
        "            input_dim=sequence_length, output_dim=output_dim)\n",
        "        self.sequence_length = sequence_length\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "\n",
        "    def call(self, inputs):\n",
        "        length = tf.shape(inputs)[-1]\n",
        "        positions = tf.range(start=0, limit=length, delta=1)\n",
        "        embedded_tokens = self.token_embeddings(inputs)\n",
        "        embedded_positions = self.position_embeddings(positions)\n",
        "        return embedded_tokens + embedded_positions\n",
        "\n",
        "    def compute_mask(self, inputs, mask=None):\n",
        "        return tf.math.not_equal(inputs, 0)\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super(PositionalEmbedding, self).get_config()\n",
        "        config.update({\n",
        "            \"output_dim\": self.output_dim,\n",
        "            \"sequence_length\": self.sequence_length,\n",
        "            \"input_dim\": self.input_dim,\n",
        "        })\n",
        "        return config"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1H7QlHwQC6Yp"
      },
      "source": [
        "**엔드-투-엔드 트랜스포머**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "vIM5VwfzN4Co"
      },
      "outputs": [],
      "source": [
        "class TransformerEncoder(layers.Layer):\n",
        "    def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        self.embed_dim = embed_dim\n",
        "        self.dense_dim = dense_dim\n",
        "        self.num_heads = num_heads\n",
        "        self.attention = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim)\n",
        "        self.dense_proj = keras.Sequential(\n",
        "            [layers.Dense(dense_dim, activation=\"relu\"),\n",
        "             layers.Dense(embed_dim),]\n",
        "        )\n",
        "        self.layernorm_1 = layers.LayerNormalization()\n",
        "        self.layernorm_2 = layers.LayerNormalization()\n",
        "\n",
        "    def call(self, inputs, mask=None):\n",
        "        if mask is not None:\n",
        "            mask = mask[:, tf.newaxis, :]\n",
        "        attention_output = self.attention(\n",
        "            inputs, inputs, attention_mask=mask)\n",
        "        proj_input = self.layernorm_1(inputs + attention_output)\n",
        "        proj_output = self.dense_proj(proj_input)\n",
        "        return self.layernorm_2(proj_input + proj_output)\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super().get_config()\n",
        "        config.update({\n",
        "            \"embed_dim\": self.embed_dim,\n",
        "            \"num_heads\": self.num_heads,\n",
        "            \"dense_dim\": self.dense_dim,\n",
        "        })\n",
        "        return config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "M-s0ZCSQC6Yp"
      },
      "outputs": [],
      "source": [
        "embed_dim = 256\n",
        "dense_dim = 2048\n",
        "num_heads = 8\n",
        "\n",
        "encoder_inputs = keras.Input(shape=(None,), dtype=\"int64\", name=\"english\")\n",
        "x = PositionalEmbedding(sequence_length, vocab_size, embed_dim)(encoder_inputs)\n",
        "encoder_outputs = TransformerEncoder(embed_dim, dense_dim, num_heads)(x)\n",
        "\n",
        "decoder_inputs = keras.Input(shape=(None,), dtype=\"int64\", name=\"spanish\")\n",
        "x = PositionalEmbedding(sequence_length, vocab_size, embed_dim)(decoder_inputs)\n",
        "x = TransformerDecoder(embed_dim, dense_dim, num_heads)(x, encoder_outputs)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "decoder_outputs = layers.Dense(vocab_size, activation=\"softmax\")(x)\n",
        "transformer = keras.Model([encoder_inputs, decoder_inputs], decoder_outputs)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IstNU4guC6Yp"
      },
      "source": [
        "**시퀀스-투-시퀀스 트랜스포머 훈련하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ox_6aZfxC6Yq",
        "outputId": "0be63774-2792-4198-95d1-9a6d29ec299e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/30\n",
            "1302/1302 [==============================] - 36s 24ms/step - loss: 1.6779 - accuracy: 0.4171 - val_loss: 1.3447 - val_accuracy: 0.5051\n",
            "Epoch 2/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 1.3466 - accuracy: 0.5288 - val_loss: 1.1721 - val_accuracy: 0.5688\n",
            "Epoch 3/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 1.2010 - accuracy: 0.5763 - val_loss: 1.1025 - val_accuracy: 0.5976\n",
            "Epoch 4/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 1.1111 - accuracy: 0.6079 - val_loss: 1.0537 - val_accuracy: 0.6196\n",
            "Epoch 5/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 1.0607 - accuracy: 0.6307 - val_loss: 1.0273 - val_accuracy: 0.6321\n",
            "Epoch 6/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 1.0279 - accuracy: 0.6475 - val_loss: 1.0111 - val_accuracy: 0.6410\n",
            "Epoch 7/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 1.0052 - accuracy: 0.6605 - val_loss: 1.0050 - val_accuracy: 0.6466\n",
            "Epoch 8/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.9865 - accuracy: 0.6707 - val_loss: 0.9997 - val_accuracy: 0.6511\n",
            "Epoch 9/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.9689 - accuracy: 0.6805 - val_loss: 0.9981 - val_accuracy: 0.6526\n",
            "Epoch 10/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.9531 - accuracy: 0.6891 - val_loss: 1.0010 - val_accuracy: 0.6566\n",
            "Epoch 11/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.9394 - accuracy: 0.6959 - val_loss: 1.0003 - val_accuracy: 0.6556\n",
            "Epoch 12/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.9264 - accuracy: 0.7020 - val_loss: 0.9991 - val_accuracy: 0.6597\n",
            "Epoch 13/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.9128 - accuracy: 0.7081 - val_loss: 1.0073 - val_accuracy: 0.6598\n",
            "Epoch 14/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8998 - accuracy: 0.7138 - val_loss: 1.0076 - val_accuracy: 0.6611\n",
            "Epoch 15/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8889 - accuracy: 0.7189 - val_loss: 1.0077 - val_accuracy: 0.6615\n",
            "Epoch 16/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8772 - accuracy: 0.7236 - val_loss: 1.0082 - val_accuracy: 0.6635\n",
            "Epoch 17/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8657 - accuracy: 0.7275 - val_loss: 1.0121 - val_accuracy: 0.6629\n",
            "Epoch 18/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8549 - accuracy: 0.7317 - val_loss: 1.0130 - val_accuracy: 0.6628\n",
            "Epoch 19/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8447 - accuracy: 0.7359 - val_loss: 1.0268 - val_accuracy: 0.6646\n",
            "Epoch 20/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8350 - accuracy: 0.7394 - val_loss: 1.0216 - val_accuracy: 0.6658\n",
            "Epoch 21/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8262 - accuracy: 0.7431 - val_loss: 1.0297 - val_accuracy: 0.6629\n",
            "Epoch 22/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8181 - accuracy: 0.7461 - val_loss: 1.0336 - val_accuracy: 0.6636\n",
            "Epoch 23/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8095 - accuracy: 0.7499 - val_loss: 1.0429 - val_accuracy: 0.6633\n",
            "Epoch 24/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.8012 - accuracy: 0.7529 - val_loss: 1.0454 - val_accuracy: 0.6640\n",
            "Epoch 25/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.7945 - accuracy: 0.7552 - val_loss: 1.0516 - val_accuracy: 0.6633\n",
            "Epoch 26/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.7869 - accuracy: 0.7586 - val_loss: 1.0520 - val_accuracy: 0.6667\n",
            "Epoch 27/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.7801 - accuracy: 0.7610 - val_loss: 1.0549 - val_accuracy: 0.6652\n",
            "Epoch 28/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.7734 - accuracy: 0.7637 - val_loss: 1.0695 - val_accuracy: 0.6643\n",
            "Epoch 29/30\n",
            "1302/1302 [==============================] - 32s 24ms/step - loss: 0.7670 - accuracy: 0.7666 - val_loss: 1.0754 - val_accuracy: 0.6622\n",
            "Epoch 30/30\n",
            "1302/1302 [==============================] - 31s 24ms/step - loss: 0.7613 - accuracy: 0.7683 - val_loss: 1.0760 - val_accuracy: 0.6636\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fa9c3f45b50>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "transformer.compile(\n",
        "    optimizer=\"rmsprop\",\n",
        "    loss=\"sparse_categorical_crossentropy\",\n",
        "    metrics=[\"accuracy\"])\n",
        "transformer.fit(train_ds, epochs=30, validation_data=val_ds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-rE4EM5PC6Yq"
      },
      "source": [
        "**트랜스포머 모델을 사용해 새로운 문장 번역하기**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gQ3O0QCLC6Yq",
        "outputId": "c5a14a3c-ccee-467e-8cd3-d4df46940143"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-\n",
            "Our electric heater did not work well.\n",
            "[start] nuestra el su [end]\n",
            "-\n",
            "All my efforts were in vain.\n",
            "[start] todos mis [UNK] fueron en primer parte [end]\n",
            "-\n",
            "They meant well.\n",
            "[start] ellos [UNK] bien [end]\n",
            "-\n",
            "Tom and Mary each took one cookie.\n",
            "[start] tom y mary [UNK] una la el que se [UNK] [end]\n",
            "-\n",
            "I gave some water to the dogs.\n",
            "[start] le di algo de agua al perro [end]\n",
            "-\n",
            "Tom wished that his classmates would treat each other with more respect.\n",
            "[start] tom [UNK] que sus esfuerzos [UNK] uno después del otro [end]\n",
            "-\n",
            "I was at the theater.\n",
            "[start] estaba en el usted punto [end]\n",
            "-\n",
            "This is the best camera in the store.\n",
            "[start] esta es la mejor cámara de fotos [end]\n",
            "-\n",
            "Let me feel it.\n",
            "[start] dime [end]\n",
            "-\n",
            "Put your hat on.\n",
            "[start] [UNK] este sombrero [end]\n",
            "-\n",
            "I had hoped that my mother would live until I got married.\n",
            "[start] esperaba haber estado tan parado aquí después viven hasta que me [UNK] [end]\n",
            "-\n",
            "Then what happened?\n",
            "[start] entonces qué ocurrió [end]\n",
            "-\n",
            "Isn't that right?\n",
            "[start] eso es verdad [end]\n",
            "-\n",
            "Tom has forgiven Mary for everything.\n",
            "[start] tom le dio a mary por todo [end]\n",
            "-\n",
            "I'll be reading a book.\n",
            "[start] voy a estar leyendo un libro [end]\n",
            "-\n",
            "Be brief.\n",
            "[start] sé [UNK] [end]\n",
            "-\n",
            "Let's see if we can get this door open.\n",
            "[start] a ver si podemos [UNK] esta puerta [end]\n",
            "-\n",
            "I think I've forgotten something.\n",
            "[start] creo que he dejado algo de que me he dejado algo [end]\n",
            "-\n",
            "He parked his car behind the building.\n",
            "[start] Él se lo [UNK] entre su coche [end]\n",
            "-\n",
            "There are no witnesses.\n",
            "[start] no hay enemigos [end]\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "spa_vocab = target_vectorization.get_vocabulary()\n",
        "spa_index_lookup = dict(zip(range(len(spa_vocab)), spa_vocab))\n",
        "max_decoded_sentence_length = 20\n",
        "\n",
        "def decode_sequence(input_sentence):\n",
        "    tokenized_input_sentence = source_vectorization([input_sentence])\n",
        "    decoded_sentence = \"[start]\"\n",
        "    for i in range(max_decoded_sentence_length):\n",
        "        tokenized_target_sentence = target_vectorization(\n",
        "            [decoded_sentence])[:, :-1]\n",
        "        predictions = transformer(\n",
        "            [tokenized_input_sentence, tokenized_target_sentence])\n",
        "        sampled_token_index = np.argmax(predictions[0, i, :])\n",
        "        sampled_token = spa_index_lookup[sampled_token_index]\n",
        "        decoded_sentence += \" \" + sampled_token\n",
        "        if sampled_token == \"[end]\":\n",
        "            break\n",
        "    return decoded_sentence\n",
        "\n",
        "test_eng_texts = [pair[0] for pair in test_pairs]\n",
        "for _ in range(20):\n",
        "    input_sentence = random.choice(test_eng_texts)\n",
        "    print(\"-\")\n",
        "    print(input_sentence)\n",
        "    print(decode_sequence(input_sentence))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gz8VRatlC6Yq"
      },
      "source": [
        "## 요약"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "chapter11_part04_sequence-to-sequence-learning.i",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "default:Python",
      "language": "python",
      "name": "conda-env-default-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.10"
    },
    "gpuClass": "premium"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}